1 /* 2 FUSE: Filesystem in Userspace 3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> 4 5 This program can be distributed under the terms of the GNU GPL. 6 See the file COPYING. 7 */ 8 9 #include "fuse_i.h" 10 11 #include <linux/pagemap.h> 12 #include <linux/slab.h> 13 #include <linux/kernel.h> 14 #include <linux/sched.h> 15 #include <linux/sched/signal.h> 16 #include <linux/module.h> 17 #include <linux/compat.h> 18 #include <linux/swap.h> 19 #include <linux/falloc.h> 20 #include <linux/uio.h> 21 22 static const struct file_operations fuse_direct_io_file_operations; 23 24 static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file, 25 int opcode, struct fuse_open_out *outargp) 26 { 27 struct fuse_open_in inarg; 28 FUSE_ARGS(args); 29 30 memset(&inarg, 0, sizeof(inarg)); 31 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); 32 if (!fc->atomic_o_trunc) 33 inarg.flags &= ~O_TRUNC; 34 args.in.h.opcode = opcode; 35 args.in.h.nodeid = nodeid; 36 args.in.numargs = 1; 37 args.in.args[0].size = sizeof(inarg); 38 args.in.args[0].value = &inarg; 39 args.out.numargs = 1; 40 args.out.args[0].size = sizeof(*outargp); 41 args.out.args[0].value = outargp; 42 43 return fuse_simple_request(fc, &args); 44 } 45 46 struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) 47 { 48 struct fuse_file *ff; 49 50 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL); 51 if (unlikely(!ff)) 52 return NULL; 53 54 ff->fc = fc; 55 ff->reserved_req = fuse_request_alloc(0); 56 if (unlikely(!ff->reserved_req)) { 57 kfree(ff); 58 return NULL; 59 } 60 61 INIT_LIST_HEAD(&ff->write_entry); 62 mutex_init(&ff->readdir.lock); 63 refcount_set(&ff->count, 1); 64 RB_CLEAR_NODE(&ff->polled_node); 65 init_waitqueue_head(&ff->poll_wait); 66 67 spin_lock(&fc->lock); 68 ff->kh = ++fc->khctr; 69 spin_unlock(&fc->lock); 70 71 return ff; 72 } 73 74 void fuse_file_free(struct fuse_file *ff) 75 { 76 fuse_request_free(ff->reserved_req); 77 mutex_destroy(&ff->readdir.lock); 78 kfree(ff); 79 } 80 81 static struct fuse_file *fuse_file_get(struct fuse_file *ff) 82 { 83 refcount_inc(&ff->count); 84 return ff; 85 } 86 87 static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req) 88 { 89 iput(req->misc.release.inode); 90 } 91 92 static void fuse_file_put(struct fuse_file *ff, bool sync) 93 { 94 if (refcount_dec_and_test(&ff->count)) { 95 struct fuse_req *req = ff->reserved_req; 96 97 if (ff->fc->no_open) { 98 /* 99 * Drop the release request when client does not 100 * implement 'open' 101 */ 102 __clear_bit(FR_BACKGROUND, &req->flags); 103 iput(req->misc.release.inode); 104 fuse_put_request(ff->fc, req); 105 } else if (sync) { 106 __set_bit(FR_FORCE, &req->flags); 107 __clear_bit(FR_BACKGROUND, &req->flags); 108 fuse_request_send(ff->fc, req); 109 iput(req->misc.release.inode); 110 fuse_put_request(ff->fc, req); 111 } else { 112 req->end = fuse_release_end; 113 __set_bit(FR_BACKGROUND, &req->flags); 114 fuse_request_send_background(ff->fc, req); 115 } 116 kfree(ff); 117 } 118 } 119 120 int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file, 121 bool isdir) 122 { 123 struct fuse_file *ff; 124 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; 125 126 ff = fuse_file_alloc(fc); 127 if (!ff) 128 return -ENOMEM; 129 130 ff->fh = 0; 131 ff->open_flags = FOPEN_KEEP_CACHE; /* Default for no-open */ 132 if (!fc->no_open || isdir) { 133 struct fuse_open_out outarg; 134 int err; 135 136 err = fuse_send_open(fc, nodeid, file, opcode, &outarg); 137 if (!err) { 138 ff->fh = outarg.fh; 139 ff->open_flags = outarg.open_flags; 140 141 } else if (err != -ENOSYS || isdir) { 142 fuse_file_free(ff); 143 return err; 144 } else { 145 fc->no_open = 1; 146 } 147 } 148 149 if (isdir) 150 ff->open_flags &= ~FOPEN_DIRECT_IO; 151 152 ff->nodeid = nodeid; 153 file->private_data = ff; 154 155 return 0; 156 } 157 EXPORT_SYMBOL_GPL(fuse_do_open); 158 159 static void fuse_link_write_file(struct file *file) 160 { 161 struct inode *inode = file_inode(file); 162 struct fuse_conn *fc = get_fuse_conn(inode); 163 struct fuse_inode *fi = get_fuse_inode(inode); 164 struct fuse_file *ff = file->private_data; 165 /* 166 * file may be written through mmap, so chain it onto the 167 * inodes's write_file list 168 */ 169 spin_lock(&fc->lock); 170 if (list_empty(&ff->write_entry)) 171 list_add(&ff->write_entry, &fi->write_files); 172 spin_unlock(&fc->lock); 173 } 174 175 void fuse_finish_open(struct inode *inode, struct file *file) 176 { 177 struct fuse_file *ff = file->private_data; 178 struct fuse_conn *fc = get_fuse_conn(inode); 179 180 if (ff->open_flags & FOPEN_DIRECT_IO) 181 file->f_op = &fuse_direct_io_file_operations; 182 if (!(ff->open_flags & FOPEN_KEEP_CACHE)) 183 invalidate_inode_pages2(inode->i_mapping); 184 if (ff->open_flags & FOPEN_NONSEEKABLE) 185 nonseekable_open(inode, file); 186 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) { 187 struct fuse_inode *fi = get_fuse_inode(inode); 188 189 spin_lock(&fc->lock); 190 fi->attr_version = ++fc->attr_version; 191 i_size_write(inode, 0); 192 spin_unlock(&fc->lock); 193 fuse_invalidate_attr(inode); 194 if (fc->writeback_cache) 195 file_update_time(file); 196 } 197 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) 198 fuse_link_write_file(file); 199 } 200 201 int fuse_open_common(struct inode *inode, struct file *file, bool isdir) 202 { 203 struct fuse_conn *fc = get_fuse_conn(inode); 204 int err; 205 bool lock_inode = (file->f_flags & O_TRUNC) && 206 fc->atomic_o_trunc && 207 fc->writeback_cache; 208 209 err = generic_file_open(inode, file); 210 if (err) 211 return err; 212 213 if (lock_inode) 214 inode_lock(inode); 215 216 err = fuse_do_open(fc, get_node_id(inode), file, isdir); 217 218 if (!err) 219 fuse_finish_open(inode, file); 220 221 if (lock_inode) 222 inode_unlock(inode); 223 224 return err; 225 } 226 227 static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode) 228 { 229 struct fuse_conn *fc = ff->fc; 230 struct fuse_req *req = ff->reserved_req; 231 struct fuse_release_in *inarg = &req->misc.release.in; 232 233 spin_lock(&fc->lock); 234 list_del(&ff->write_entry); 235 if (!RB_EMPTY_NODE(&ff->polled_node)) 236 rb_erase(&ff->polled_node, &fc->polled_files); 237 spin_unlock(&fc->lock); 238 239 wake_up_interruptible_all(&ff->poll_wait); 240 241 inarg->fh = ff->fh; 242 inarg->flags = flags; 243 req->in.h.opcode = opcode; 244 req->in.h.nodeid = ff->nodeid; 245 req->in.numargs = 1; 246 req->in.args[0].size = sizeof(struct fuse_release_in); 247 req->in.args[0].value = inarg; 248 } 249 250 void fuse_release_common(struct file *file, int opcode) 251 { 252 struct fuse_file *ff = file->private_data; 253 struct fuse_req *req = ff->reserved_req; 254 255 fuse_prepare_release(ff, file->f_flags, opcode); 256 257 if (ff->flock) { 258 struct fuse_release_in *inarg = &req->misc.release.in; 259 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; 260 inarg->lock_owner = fuse_lock_owner_id(ff->fc, 261 (fl_owner_t) file); 262 } 263 /* Hold inode until release is finished */ 264 req->misc.release.inode = igrab(file_inode(file)); 265 266 /* 267 * Normally this will send the RELEASE request, however if 268 * some asynchronous READ or WRITE requests are outstanding, 269 * the sending will be delayed. 270 * 271 * Make the release synchronous if this is a fuseblk mount, 272 * synchronous RELEASE is allowed (and desirable) in this case 273 * because the server can be trusted not to screw up. 274 */ 275 fuse_file_put(ff, ff->fc->destroy_req != NULL); 276 } 277 278 static int fuse_open(struct inode *inode, struct file *file) 279 { 280 return fuse_open_common(inode, file, false); 281 } 282 283 static int fuse_release(struct inode *inode, struct file *file) 284 { 285 struct fuse_conn *fc = get_fuse_conn(inode); 286 287 /* see fuse_vma_close() for !writeback_cache case */ 288 if (fc->writeback_cache) 289 write_inode_now(inode, 1); 290 291 fuse_release_common(file, FUSE_RELEASE); 292 293 /* return value is ignored by VFS */ 294 return 0; 295 } 296 297 void fuse_sync_release(struct fuse_file *ff, int flags) 298 { 299 WARN_ON(refcount_read(&ff->count) > 1); 300 fuse_prepare_release(ff, flags, FUSE_RELEASE); 301 /* 302 * iput(NULL) is a no-op and since the refcount is 1 and everything's 303 * synchronous, we are fine with not doing igrab() here" 304 */ 305 fuse_file_put(ff, true); 306 } 307 EXPORT_SYMBOL_GPL(fuse_sync_release); 308 309 /* 310 * Scramble the ID space with XTEA, so that the value of the files_struct 311 * pointer is not exposed to userspace. 312 */ 313 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) 314 { 315 u32 *k = fc->scramble_key; 316 u64 v = (unsigned long) id; 317 u32 v0 = v; 318 u32 v1 = v >> 32; 319 u32 sum = 0; 320 int i; 321 322 for (i = 0; i < 32; i++) { 323 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]); 324 sum += 0x9E3779B9; 325 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]); 326 } 327 328 return (u64) v0 + ((u64) v1 << 32); 329 } 330 331 /* 332 * Check if any page in a range is under writeback 333 * 334 * This is currently done by walking the list of writepage requests 335 * for the inode, which can be pretty inefficient. 336 */ 337 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from, 338 pgoff_t idx_to) 339 { 340 struct fuse_conn *fc = get_fuse_conn(inode); 341 struct fuse_inode *fi = get_fuse_inode(inode); 342 struct fuse_req *req; 343 bool found = false; 344 345 spin_lock(&fc->lock); 346 list_for_each_entry(req, &fi->writepages, writepages_entry) { 347 pgoff_t curr_index; 348 349 BUG_ON(req->inode != inode); 350 curr_index = req->misc.write.in.offset >> PAGE_SHIFT; 351 if (idx_from < curr_index + req->num_pages && 352 curr_index <= idx_to) { 353 found = true; 354 break; 355 } 356 } 357 spin_unlock(&fc->lock); 358 359 return found; 360 } 361 362 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index) 363 { 364 return fuse_range_is_writeback(inode, index, index); 365 } 366 367 /* 368 * Wait for page writeback to be completed. 369 * 370 * Since fuse doesn't rely on the VM writeback tracking, this has to 371 * use some other means. 372 */ 373 static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) 374 { 375 struct fuse_inode *fi = get_fuse_inode(inode); 376 377 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); 378 return 0; 379 } 380 381 /* 382 * Wait for all pending writepages on the inode to finish. 383 * 384 * This is currently done by blocking further writes with FUSE_NOWRITE 385 * and waiting for all sent writes to complete. 386 * 387 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage 388 * could conflict with truncation. 389 */ 390 static void fuse_sync_writes(struct inode *inode) 391 { 392 fuse_set_nowrite(inode); 393 fuse_release_nowrite(inode); 394 } 395 396 static int fuse_flush(struct file *file, fl_owner_t id) 397 { 398 struct inode *inode = file_inode(file); 399 struct fuse_conn *fc = get_fuse_conn(inode); 400 struct fuse_file *ff = file->private_data; 401 struct fuse_req *req; 402 struct fuse_flush_in inarg; 403 int err; 404 405 if (is_bad_inode(inode)) 406 return -EIO; 407 408 if (fc->no_flush) 409 return 0; 410 411 err = write_inode_now(inode, 1); 412 if (err) 413 return err; 414 415 inode_lock(inode); 416 fuse_sync_writes(inode); 417 inode_unlock(inode); 418 419 err = filemap_check_errors(file->f_mapping); 420 if (err) 421 return err; 422 423 req = fuse_get_req_nofail_nopages(fc, file); 424 memset(&inarg, 0, sizeof(inarg)); 425 inarg.fh = ff->fh; 426 inarg.lock_owner = fuse_lock_owner_id(fc, id); 427 req->in.h.opcode = FUSE_FLUSH; 428 req->in.h.nodeid = get_node_id(inode); 429 req->in.numargs = 1; 430 req->in.args[0].size = sizeof(inarg); 431 req->in.args[0].value = &inarg; 432 __set_bit(FR_FORCE, &req->flags); 433 fuse_request_send(fc, req); 434 err = req->out.h.error; 435 fuse_put_request(fc, req); 436 if (err == -ENOSYS) { 437 fc->no_flush = 1; 438 err = 0; 439 } 440 return err; 441 } 442 443 int fuse_fsync_common(struct file *file, loff_t start, loff_t end, 444 int datasync, int isdir) 445 { 446 struct inode *inode = file->f_mapping->host; 447 struct fuse_conn *fc = get_fuse_conn(inode); 448 struct fuse_file *ff = file->private_data; 449 FUSE_ARGS(args); 450 struct fuse_fsync_in inarg; 451 int err; 452 453 if (is_bad_inode(inode)) 454 return -EIO; 455 456 inode_lock(inode); 457 458 /* 459 * Start writeback against all dirty pages of the inode, then 460 * wait for all outstanding writes, before sending the FSYNC 461 * request. 462 */ 463 err = file_write_and_wait_range(file, start, end); 464 if (err) 465 goto out; 466 467 fuse_sync_writes(inode); 468 469 /* 470 * Due to implementation of fuse writeback 471 * file_write_and_wait_range() does not catch errors. 472 * We have to do this directly after fuse_sync_writes() 473 */ 474 err = file_check_and_advance_wb_err(file); 475 if (err) 476 goto out; 477 478 err = sync_inode_metadata(inode, 1); 479 if (err) 480 goto out; 481 482 if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir)) 483 goto out; 484 485 memset(&inarg, 0, sizeof(inarg)); 486 inarg.fh = ff->fh; 487 inarg.fsync_flags = datasync ? 1 : 0; 488 args.in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC; 489 args.in.h.nodeid = get_node_id(inode); 490 args.in.numargs = 1; 491 args.in.args[0].size = sizeof(inarg); 492 args.in.args[0].value = &inarg; 493 err = fuse_simple_request(fc, &args); 494 if (err == -ENOSYS) { 495 if (isdir) 496 fc->no_fsyncdir = 1; 497 else 498 fc->no_fsync = 1; 499 err = 0; 500 } 501 out: 502 inode_unlock(inode); 503 return err; 504 } 505 506 static int fuse_fsync(struct file *file, loff_t start, loff_t end, 507 int datasync) 508 { 509 return fuse_fsync_common(file, start, end, datasync, 0); 510 } 511 512 void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos, 513 size_t count, int opcode) 514 { 515 struct fuse_read_in *inarg = &req->misc.read.in; 516 struct fuse_file *ff = file->private_data; 517 518 inarg->fh = ff->fh; 519 inarg->offset = pos; 520 inarg->size = count; 521 inarg->flags = file->f_flags; 522 req->in.h.opcode = opcode; 523 req->in.h.nodeid = ff->nodeid; 524 req->in.numargs = 1; 525 req->in.args[0].size = sizeof(struct fuse_read_in); 526 req->in.args[0].value = inarg; 527 req->out.argvar = 1; 528 req->out.numargs = 1; 529 req->out.args[0].size = count; 530 } 531 532 static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty) 533 { 534 unsigned i; 535 536 for (i = 0; i < req->num_pages; i++) { 537 struct page *page = req->pages[i]; 538 if (should_dirty) 539 set_page_dirty_lock(page); 540 put_page(page); 541 } 542 } 543 544 static void fuse_io_release(struct kref *kref) 545 { 546 kfree(container_of(kref, struct fuse_io_priv, refcnt)); 547 } 548 549 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) 550 { 551 if (io->err) 552 return io->err; 553 554 if (io->bytes >= 0 && io->write) 555 return -EIO; 556 557 return io->bytes < 0 ? io->size : io->bytes; 558 } 559 560 /** 561 * In case of short read, the caller sets 'pos' to the position of 562 * actual end of fuse request in IO request. Otherwise, if bytes_requested 563 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1. 564 * 565 * An example: 566 * User requested DIO read of 64K. It was splitted into two 32K fuse requests, 567 * both submitted asynchronously. The first of them was ACKed by userspace as 568 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The 569 * second request was ACKed as short, e.g. only 1K was read, resulting in 570 * pos == 33K. 571 * 572 * Thus, when all fuse requests are completed, the minimal non-negative 'pos' 573 * will be equal to the length of the longest contiguous fragment of 574 * transferred data starting from the beginning of IO request. 575 */ 576 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) 577 { 578 int left; 579 580 spin_lock(&io->lock); 581 if (err) 582 io->err = io->err ? : err; 583 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes)) 584 io->bytes = pos; 585 586 left = --io->reqs; 587 if (!left && io->blocking) 588 complete(io->done); 589 spin_unlock(&io->lock); 590 591 if (!left && !io->blocking) { 592 ssize_t res = fuse_get_res_by_io(io); 593 594 if (res >= 0) { 595 struct inode *inode = file_inode(io->iocb->ki_filp); 596 struct fuse_conn *fc = get_fuse_conn(inode); 597 struct fuse_inode *fi = get_fuse_inode(inode); 598 599 spin_lock(&fc->lock); 600 fi->attr_version = ++fc->attr_version; 601 spin_unlock(&fc->lock); 602 } 603 604 io->iocb->ki_complete(io->iocb, res, 0); 605 } 606 607 kref_put(&io->refcnt, fuse_io_release); 608 } 609 610 static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req) 611 { 612 struct fuse_io_priv *io = req->io; 613 ssize_t pos = -1; 614 615 fuse_release_user_pages(req, io->should_dirty); 616 617 if (io->write) { 618 if (req->misc.write.in.size != req->misc.write.out.size) 619 pos = req->misc.write.in.offset - io->offset + 620 req->misc.write.out.size; 621 } else { 622 if (req->misc.read.in.size != req->out.args[0].size) 623 pos = req->misc.read.in.offset - io->offset + 624 req->out.args[0].size; 625 } 626 627 fuse_aio_complete(io, req->out.h.error, pos); 628 } 629 630 static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req, 631 size_t num_bytes, struct fuse_io_priv *io) 632 { 633 spin_lock(&io->lock); 634 kref_get(&io->refcnt); 635 io->size += num_bytes; 636 io->reqs++; 637 spin_unlock(&io->lock); 638 639 req->io = io; 640 req->end = fuse_aio_complete_req; 641 642 __fuse_get_request(req); 643 fuse_request_send_background(fc, req); 644 645 return num_bytes; 646 } 647 648 static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io, 649 loff_t pos, size_t count, fl_owner_t owner) 650 { 651 struct file *file = io->iocb->ki_filp; 652 struct fuse_file *ff = file->private_data; 653 struct fuse_conn *fc = ff->fc; 654 655 fuse_read_fill(req, file, pos, count, FUSE_READ); 656 if (owner != NULL) { 657 struct fuse_read_in *inarg = &req->misc.read.in; 658 659 inarg->read_flags |= FUSE_READ_LOCKOWNER; 660 inarg->lock_owner = fuse_lock_owner_id(fc, owner); 661 } 662 663 if (io->async) 664 return fuse_async_req_send(fc, req, count, io); 665 666 fuse_request_send(fc, req); 667 return req->out.args[0].size; 668 } 669 670 static void fuse_read_update_size(struct inode *inode, loff_t size, 671 u64 attr_ver) 672 { 673 struct fuse_conn *fc = get_fuse_conn(inode); 674 struct fuse_inode *fi = get_fuse_inode(inode); 675 676 spin_lock(&fc->lock); 677 if (attr_ver == fi->attr_version && size < inode->i_size && 678 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { 679 fi->attr_version = ++fc->attr_version; 680 i_size_write(inode, size); 681 } 682 spin_unlock(&fc->lock); 683 } 684 685 static void fuse_short_read(struct fuse_req *req, struct inode *inode, 686 u64 attr_ver) 687 { 688 size_t num_read = req->out.args[0].size; 689 struct fuse_conn *fc = get_fuse_conn(inode); 690 691 if (fc->writeback_cache) { 692 /* 693 * A hole in a file. Some data after the hole are in page cache, 694 * but have not reached the client fs yet. So, the hole is not 695 * present there. 696 */ 697 int i; 698 int start_idx = num_read >> PAGE_SHIFT; 699 size_t off = num_read & (PAGE_SIZE - 1); 700 701 for (i = start_idx; i < req->num_pages; i++) { 702 zero_user_segment(req->pages[i], off, PAGE_SIZE); 703 off = 0; 704 } 705 } else { 706 loff_t pos = page_offset(req->pages[0]) + num_read; 707 fuse_read_update_size(inode, pos, attr_ver); 708 } 709 } 710 711 static int fuse_do_readpage(struct file *file, struct page *page) 712 { 713 struct kiocb iocb; 714 struct fuse_io_priv io; 715 struct inode *inode = page->mapping->host; 716 struct fuse_conn *fc = get_fuse_conn(inode); 717 struct fuse_req *req; 718 size_t num_read; 719 loff_t pos = page_offset(page); 720 size_t count = PAGE_SIZE; 721 u64 attr_ver; 722 int err; 723 724 /* 725 * Page writeback can extend beyond the lifetime of the 726 * page-cache page, so make sure we read a properly synced 727 * page. 728 */ 729 fuse_wait_on_page_writeback(inode, page->index); 730 731 req = fuse_get_req(fc, 1); 732 if (IS_ERR(req)) 733 return PTR_ERR(req); 734 735 attr_ver = fuse_get_attr_version(fc); 736 737 req->out.page_zeroing = 1; 738 req->out.argpages = 1; 739 req->num_pages = 1; 740 req->pages[0] = page; 741 req->page_descs[0].length = count; 742 init_sync_kiocb(&iocb, file); 743 io = (struct fuse_io_priv) FUSE_IO_PRIV_SYNC(&iocb); 744 num_read = fuse_send_read(req, &io, pos, count, NULL); 745 err = req->out.h.error; 746 747 if (!err) { 748 /* 749 * Short read means EOF. If file size is larger, truncate it 750 */ 751 if (num_read < count) 752 fuse_short_read(req, inode, attr_ver); 753 754 SetPageUptodate(page); 755 } 756 757 fuse_put_request(fc, req); 758 759 return err; 760 } 761 762 static int fuse_readpage(struct file *file, struct page *page) 763 { 764 struct inode *inode = page->mapping->host; 765 int err; 766 767 err = -EIO; 768 if (is_bad_inode(inode)) 769 goto out; 770 771 err = fuse_do_readpage(file, page); 772 fuse_invalidate_atime(inode); 773 out: 774 unlock_page(page); 775 return err; 776 } 777 778 static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req) 779 { 780 int i; 781 size_t count = req->misc.read.in.size; 782 size_t num_read = req->out.args[0].size; 783 struct address_space *mapping = NULL; 784 785 for (i = 0; mapping == NULL && i < req->num_pages; i++) 786 mapping = req->pages[i]->mapping; 787 788 if (mapping) { 789 struct inode *inode = mapping->host; 790 791 /* 792 * Short read means EOF. If file size is larger, truncate it 793 */ 794 if (!req->out.h.error && num_read < count) 795 fuse_short_read(req, inode, req->misc.read.attr_ver); 796 797 fuse_invalidate_atime(inode); 798 } 799 800 for (i = 0; i < req->num_pages; i++) { 801 struct page *page = req->pages[i]; 802 if (!req->out.h.error) 803 SetPageUptodate(page); 804 else 805 SetPageError(page); 806 unlock_page(page); 807 put_page(page); 808 } 809 if (req->ff) 810 fuse_file_put(req->ff, false); 811 } 812 813 static void fuse_send_readpages(struct fuse_req *req, struct file *file) 814 { 815 struct fuse_file *ff = file->private_data; 816 struct fuse_conn *fc = ff->fc; 817 loff_t pos = page_offset(req->pages[0]); 818 size_t count = req->num_pages << PAGE_SHIFT; 819 820 req->out.argpages = 1; 821 req->out.page_zeroing = 1; 822 req->out.page_replace = 1; 823 fuse_read_fill(req, file, pos, count, FUSE_READ); 824 req->misc.read.attr_ver = fuse_get_attr_version(fc); 825 if (fc->async_read) { 826 req->ff = fuse_file_get(ff); 827 req->end = fuse_readpages_end; 828 fuse_request_send_background(fc, req); 829 } else { 830 fuse_request_send(fc, req); 831 fuse_readpages_end(fc, req); 832 fuse_put_request(fc, req); 833 } 834 } 835 836 struct fuse_fill_data { 837 struct fuse_req *req; 838 struct file *file; 839 struct inode *inode; 840 unsigned nr_pages; 841 }; 842 843 static int fuse_readpages_fill(void *_data, struct page *page) 844 { 845 struct fuse_fill_data *data = _data; 846 struct fuse_req *req = data->req; 847 struct inode *inode = data->inode; 848 struct fuse_conn *fc = get_fuse_conn(inode); 849 850 fuse_wait_on_page_writeback(inode, page->index); 851 852 if (req->num_pages && 853 (req->num_pages == fc->max_pages || 854 (req->num_pages + 1) * PAGE_SIZE > fc->max_read || 855 req->pages[req->num_pages - 1]->index + 1 != page->index)) { 856 unsigned int nr_alloc = min_t(unsigned int, data->nr_pages, 857 fc->max_pages); 858 fuse_send_readpages(req, data->file); 859 if (fc->async_read) 860 req = fuse_get_req_for_background(fc, nr_alloc); 861 else 862 req = fuse_get_req(fc, nr_alloc); 863 864 data->req = req; 865 if (IS_ERR(req)) { 866 unlock_page(page); 867 return PTR_ERR(req); 868 } 869 } 870 871 if (WARN_ON(req->num_pages >= req->max_pages)) { 872 unlock_page(page); 873 fuse_put_request(fc, req); 874 return -EIO; 875 } 876 877 get_page(page); 878 req->pages[req->num_pages] = page; 879 req->page_descs[req->num_pages].length = PAGE_SIZE; 880 req->num_pages++; 881 data->nr_pages--; 882 return 0; 883 } 884 885 static int fuse_readpages(struct file *file, struct address_space *mapping, 886 struct list_head *pages, unsigned nr_pages) 887 { 888 struct inode *inode = mapping->host; 889 struct fuse_conn *fc = get_fuse_conn(inode); 890 struct fuse_fill_data data; 891 int err; 892 unsigned int nr_alloc = min_t(unsigned int, nr_pages, fc->max_pages); 893 894 err = -EIO; 895 if (is_bad_inode(inode)) 896 goto out; 897 898 data.file = file; 899 data.inode = inode; 900 if (fc->async_read) 901 data.req = fuse_get_req_for_background(fc, nr_alloc); 902 else 903 data.req = fuse_get_req(fc, nr_alloc); 904 data.nr_pages = nr_pages; 905 err = PTR_ERR(data.req); 906 if (IS_ERR(data.req)) 907 goto out; 908 909 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data); 910 if (!err) { 911 if (data.req->num_pages) 912 fuse_send_readpages(data.req, file); 913 else 914 fuse_put_request(fc, data.req); 915 } 916 out: 917 return err; 918 } 919 920 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 921 { 922 struct inode *inode = iocb->ki_filp->f_mapping->host; 923 struct fuse_conn *fc = get_fuse_conn(inode); 924 925 /* 926 * In auto invalidate mode, always update attributes on read. 927 * Otherwise, only update if we attempt to read past EOF (to ensure 928 * i_size is up to date). 929 */ 930 if (fc->auto_inval_data || 931 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { 932 int err; 933 err = fuse_update_attributes(inode, iocb->ki_filp); 934 if (err) 935 return err; 936 } 937 938 return generic_file_read_iter(iocb, to); 939 } 940 941 static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff, 942 loff_t pos, size_t count) 943 { 944 struct fuse_write_in *inarg = &req->misc.write.in; 945 struct fuse_write_out *outarg = &req->misc.write.out; 946 947 inarg->fh = ff->fh; 948 inarg->offset = pos; 949 inarg->size = count; 950 req->in.h.opcode = FUSE_WRITE; 951 req->in.h.nodeid = ff->nodeid; 952 req->in.numargs = 2; 953 if (ff->fc->minor < 9) 954 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; 955 else 956 req->in.args[0].size = sizeof(struct fuse_write_in); 957 req->in.args[0].value = inarg; 958 req->in.args[1].size = count; 959 req->out.numargs = 1; 960 req->out.args[0].size = sizeof(struct fuse_write_out); 961 req->out.args[0].value = outarg; 962 } 963 964 static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io, 965 loff_t pos, size_t count, fl_owner_t owner) 966 { 967 struct kiocb *iocb = io->iocb; 968 struct file *file = iocb->ki_filp; 969 struct fuse_file *ff = file->private_data; 970 struct fuse_conn *fc = ff->fc; 971 struct fuse_write_in *inarg = &req->misc.write.in; 972 973 fuse_write_fill(req, ff, pos, count); 974 inarg->flags = file->f_flags; 975 if (iocb->ki_flags & IOCB_DSYNC) 976 inarg->flags |= O_DSYNC; 977 if (iocb->ki_flags & IOCB_SYNC) 978 inarg->flags |= O_SYNC; 979 if (owner != NULL) { 980 inarg->write_flags |= FUSE_WRITE_LOCKOWNER; 981 inarg->lock_owner = fuse_lock_owner_id(fc, owner); 982 } 983 984 if (io->async) 985 return fuse_async_req_send(fc, req, count, io); 986 987 fuse_request_send(fc, req); 988 return req->misc.write.out.size; 989 } 990 991 bool fuse_write_update_size(struct inode *inode, loff_t pos) 992 { 993 struct fuse_conn *fc = get_fuse_conn(inode); 994 struct fuse_inode *fi = get_fuse_inode(inode); 995 bool ret = false; 996 997 spin_lock(&fc->lock); 998 fi->attr_version = ++fc->attr_version; 999 if (pos > inode->i_size) { 1000 i_size_write(inode, pos); 1001 ret = true; 1002 } 1003 spin_unlock(&fc->lock); 1004 1005 return ret; 1006 } 1007 1008 static size_t fuse_send_write_pages(struct fuse_req *req, struct kiocb *iocb, 1009 struct inode *inode, loff_t pos, 1010 size_t count) 1011 { 1012 size_t res; 1013 unsigned offset; 1014 unsigned i; 1015 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1016 1017 for (i = 0; i < req->num_pages; i++) 1018 fuse_wait_on_page_writeback(inode, req->pages[i]->index); 1019 1020 res = fuse_send_write(req, &io, pos, count, NULL); 1021 1022 offset = req->page_descs[0].offset; 1023 count = res; 1024 for (i = 0; i < req->num_pages; i++) { 1025 struct page *page = req->pages[i]; 1026 1027 if (!req->out.h.error && !offset && count >= PAGE_SIZE) 1028 SetPageUptodate(page); 1029 1030 if (count > PAGE_SIZE - offset) 1031 count -= PAGE_SIZE - offset; 1032 else 1033 count = 0; 1034 offset = 0; 1035 1036 unlock_page(page); 1037 put_page(page); 1038 } 1039 1040 return res; 1041 } 1042 1043 static ssize_t fuse_fill_write_pages(struct fuse_req *req, 1044 struct address_space *mapping, 1045 struct iov_iter *ii, loff_t pos) 1046 { 1047 struct fuse_conn *fc = get_fuse_conn(mapping->host); 1048 unsigned offset = pos & (PAGE_SIZE - 1); 1049 size_t count = 0; 1050 int err; 1051 1052 req->in.argpages = 1; 1053 req->page_descs[0].offset = offset; 1054 1055 do { 1056 size_t tmp; 1057 struct page *page; 1058 pgoff_t index = pos >> PAGE_SHIFT; 1059 size_t bytes = min_t(size_t, PAGE_SIZE - offset, 1060 iov_iter_count(ii)); 1061 1062 bytes = min_t(size_t, bytes, fc->max_write - count); 1063 1064 again: 1065 err = -EFAULT; 1066 if (iov_iter_fault_in_readable(ii, bytes)) 1067 break; 1068 1069 err = -ENOMEM; 1070 page = grab_cache_page_write_begin(mapping, index, 0); 1071 if (!page) 1072 break; 1073 1074 if (mapping_writably_mapped(mapping)) 1075 flush_dcache_page(page); 1076 1077 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes); 1078 flush_dcache_page(page); 1079 1080 iov_iter_advance(ii, tmp); 1081 if (!tmp) { 1082 unlock_page(page); 1083 put_page(page); 1084 bytes = min(bytes, iov_iter_single_seg_count(ii)); 1085 goto again; 1086 } 1087 1088 err = 0; 1089 req->pages[req->num_pages] = page; 1090 req->page_descs[req->num_pages].length = tmp; 1091 req->num_pages++; 1092 1093 count += tmp; 1094 pos += tmp; 1095 offset += tmp; 1096 if (offset == PAGE_SIZE) 1097 offset = 0; 1098 1099 if (!fc->big_writes) 1100 break; 1101 } while (iov_iter_count(ii) && count < fc->max_write && 1102 req->num_pages < req->max_pages && offset == 0); 1103 1104 return count > 0 ? count : err; 1105 } 1106 1107 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len, 1108 unsigned int max_pages) 1109 { 1110 return min_t(unsigned int, 1111 ((pos + len - 1) >> PAGE_SHIFT) - 1112 (pos >> PAGE_SHIFT) + 1, 1113 max_pages); 1114 } 1115 1116 static ssize_t fuse_perform_write(struct kiocb *iocb, 1117 struct address_space *mapping, 1118 struct iov_iter *ii, loff_t pos) 1119 { 1120 struct inode *inode = mapping->host; 1121 struct fuse_conn *fc = get_fuse_conn(inode); 1122 struct fuse_inode *fi = get_fuse_inode(inode); 1123 int err = 0; 1124 ssize_t res = 0; 1125 1126 if (is_bad_inode(inode)) 1127 return -EIO; 1128 1129 if (inode->i_size < pos + iov_iter_count(ii)) 1130 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1131 1132 do { 1133 struct fuse_req *req; 1134 ssize_t count; 1135 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), 1136 fc->max_pages); 1137 1138 req = fuse_get_req(fc, nr_pages); 1139 if (IS_ERR(req)) { 1140 err = PTR_ERR(req); 1141 break; 1142 } 1143 1144 count = fuse_fill_write_pages(req, mapping, ii, pos); 1145 if (count <= 0) { 1146 err = count; 1147 } else { 1148 size_t num_written; 1149 1150 num_written = fuse_send_write_pages(req, iocb, inode, 1151 pos, count); 1152 err = req->out.h.error; 1153 if (!err) { 1154 res += num_written; 1155 pos += num_written; 1156 1157 /* break out of the loop on short write */ 1158 if (num_written != count) 1159 err = -EIO; 1160 } 1161 } 1162 fuse_put_request(fc, req); 1163 } while (!err && iov_iter_count(ii)); 1164 1165 if (res > 0) 1166 fuse_write_update_size(inode, pos); 1167 1168 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1169 fuse_invalidate_attr(inode); 1170 1171 return res > 0 ? res : err; 1172 } 1173 1174 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 1175 { 1176 struct file *file = iocb->ki_filp; 1177 struct address_space *mapping = file->f_mapping; 1178 ssize_t written = 0; 1179 ssize_t written_buffered = 0; 1180 struct inode *inode = mapping->host; 1181 ssize_t err; 1182 loff_t endbyte = 0; 1183 1184 if (get_fuse_conn(inode)->writeback_cache) { 1185 /* Update size (EOF optimization) and mode (SUID clearing) */ 1186 err = fuse_update_attributes(mapping->host, file); 1187 if (err) 1188 return err; 1189 1190 return generic_file_write_iter(iocb, from); 1191 } 1192 1193 inode_lock(inode); 1194 1195 /* We can write back this queue in page reclaim */ 1196 current->backing_dev_info = inode_to_bdi(inode); 1197 1198 err = generic_write_checks(iocb, from); 1199 if (err <= 0) 1200 goto out; 1201 1202 err = file_remove_privs(file); 1203 if (err) 1204 goto out; 1205 1206 err = file_update_time(file); 1207 if (err) 1208 goto out; 1209 1210 if (iocb->ki_flags & IOCB_DIRECT) { 1211 loff_t pos = iocb->ki_pos; 1212 written = generic_file_direct_write(iocb, from); 1213 if (written < 0 || !iov_iter_count(from)) 1214 goto out; 1215 1216 pos += written; 1217 1218 written_buffered = fuse_perform_write(iocb, mapping, from, pos); 1219 if (written_buffered < 0) { 1220 err = written_buffered; 1221 goto out; 1222 } 1223 endbyte = pos + written_buffered - 1; 1224 1225 err = filemap_write_and_wait_range(file->f_mapping, pos, 1226 endbyte); 1227 if (err) 1228 goto out; 1229 1230 invalidate_mapping_pages(file->f_mapping, 1231 pos >> PAGE_SHIFT, 1232 endbyte >> PAGE_SHIFT); 1233 1234 written += written_buffered; 1235 iocb->ki_pos = pos + written_buffered; 1236 } else { 1237 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos); 1238 if (written >= 0) 1239 iocb->ki_pos += written; 1240 } 1241 out: 1242 current->backing_dev_info = NULL; 1243 inode_unlock(inode); 1244 if (written > 0) 1245 written = generic_write_sync(iocb, written); 1246 1247 return written ? written : err; 1248 } 1249 1250 static inline void fuse_page_descs_length_init(struct fuse_req *req, 1251 unsigned index, unsigned nr_pages) 1252 { 1253 int i; 1254 1255 for (i = index; i < index + nr_pages; i++) 1256 req->page_descs[i].length = PAGE_SIZE - 1257 req->page_descs[i].offset; 1258 } 1259 1260 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) 1261 { 1262 return (unsigned long)ii->iov->iov_base + ii->iov_offset; 1263 } 1264 1265 static inline size_t fuse_get_frag_size(const struct iov_iter *ii, 1266 size_t max_size) 1267 { 1268 return min(iov_iter_single_seg_count(ii), max_size); 1269 } 1270 1271 static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii, 1272 size_t *nbytesp, int write) 1273 { 1274 size_t nbytes = 0; /* # bytes already packed in req */ 1275 ssize_t ret = 0; 1276 1277 /* Special case for kernel I/O: can copy directly into the buffer */ 1278 if (iov_iter_is_kvec(ii)) { 1279 unsigned long user_addr = fuse_get_user_addr(ii); 1280 size_t frag_size = fuse_get_frag_size(ii, *nbytesp); 1281 1282 if (write) 1283 req->in.args[1].value = (void *) user_addr; 1284 else 1285 req->out.args[0].value = (void *) user_addr; 1286 1287 iov_iter_advance(ii, frag_size); 1288 *nbytesp = frag_size; 1289 return 0; 1290 } 1291 1292 while (nbytes < *nbytesp && req->num_pages < req->max_pages) { 1293 unsigned npages; 1294 size_t start; 1295 ret = iov_iter_get_pages(ii, &req->pages[req->num_pages], 1296 *nbytesp - nbytes, 1297 req->max_pages - req->num_pages, 1298 &start); 1299 if (ret < 0) 1300 break; 1301 1302 iov_iter_advance(ii, ret); 1303 nbytes += ret; 1304 1305 ret += start; 1306 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE; 1307 1308 req->page_descs[req->num_pages].offset = start; 1309 fuse_page_descs_length_init(req, req->num_pages, npages); 1310 1311 req->num_pages += npages; 1312 req->page_descs[req->num_pages - 1].length -= 1313 (PAGE_SIZE - ret) & (PAGE_SIZE - 1); 1314 } 1315 1316 if (write) 1317 req->in.argpages = 1; 1318 else 1319 req->out.argpages = 1; 1320 1321 *nbytesp = nbytes; 1322 1323 return ret < 0 ? ret : 0; 1324 } 1325 1326 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, 1327 loff_t *ppos, int flags) 1328 { 1329 int write = flags & FUSE_DIO_WRITE; 1330 int cuse = flags & FUSE_DIO_CUSE; 1331 struct file *file = io->iocb->ki_filp; 1332 struct inode *inode = file->f_mapping->host; 1333 struct fuse_file *ff = file->private_data; 1334 struct fuse_conn *fc = ff->fc; 1335 size_t nmax = write ? fc->max_write : fc->max_read; 1336 loff_t pos = *ppos; 1337 size_t count = iov_iter_count(iter); 1338 pgoff_t idx_from = pos >> PAGE_SHIFT; 1339 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT; 1340 ssize_t res = 0; 1341 struct fuse_req *req; 1342 int err = 0; 1343 1344 if (io->async) 1345 req = fuse_get_req_for_background(fc, iov_iter_npages(iter, 1346 fc->max_pages)); 1347 else 1348 req = fuse_get_req(fc, iov_iter_npages(iter, fc->max_pages)); 1349 if (IS_ERR(req)) 1350 return PTR_ERR(req); 1351 1352 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) { 1353 if (!write) 1354 inode_lock(inode); 1355 fuse_sync_writes(inode); 1356 if (!write) 1357 inode_unlock(inode); 1358 } 1359 1360 io->should_dirty = !write && iter_is_iovec(iter); 1361 while (count) { 1362 size_t nres; 1363 fl_owner_t owner = current->files; 1364 size_t nbytes = min(count, nmax); 1365 err = fuse_get_user_pages(req, iter, &nbytes, write); 1366 if (err && !nbytes) 1367 break; 1368 1369 if (write) 1370 nres = fuse_send_write(req, io, pos, nbytes, owner); 1371 else 1372 nres = fuse_send_read(req, io, pos, nbytes, owner); 1373 1374 if (!io->async) 1375 fuse_release_user_pages(req, io->should_dirty); 1376 if (req->out.h.error) { 1377 err = req->out.h.error; 1378 break; 1379 } else if (nres > nbytes) { 1380 res = 0; 1381 err = -EIO; 1382 break; 1383 } 1384 count -= nres; 1385 res += nres; 1386 pos += nres; 1387 if (nres != nbytes) 1388 break; 1389 if (count) { 1390 fuse_put_request(fc, req); 1391 if (io->async) 1392 req = fuse_get_req_for_background(fc, 1393 iov_iter_npages(iter, fc->max_pages)); 1394 else 1395 req = fuse_get_req(fc, iov_iter_npages(iter, 1396 fc->max_pages)); 1397 if (IS_ERR(req)) 1398 break; 1399 } 1400 } 1401 if (!IS_ERR(req)) 1402 fuse_put_request(fc, req); 1403 if (res > 0) 1404 *ppos = pos; 1405 1406 return res > 0 ? res : err; 1407 } 1408 EXPORT_SYMBOL_GPL(fuse_direct_io); 1409 1410 static ssize_t __fuse_direct_read(struct fuse_io_priv *io, 1411 struct iov_iter *iter, 1412 loff_t *ppos) 1413 { 1414 ssize_t res; 1415 struct inode *inode = file_inode(io->iocb->ki_filp); 1416 1417 if (is_bad_inode(inode)) 1418 return -EIO; 1419 1420 res = fuse_direct_io(io, iter, ppos, 0); 1421 1422 fuse_invalidate_atime(inode); 1423 1424 return res; 1425 } 1426 1427 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) 1428 { 1429 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1430 return __fuse_direct_read(&io, to, &iocb->ki_pos); 1431 } 1432 1433 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) 1434 { 1435 struct inode *inode = file_inode(iocb->ki_filp); 1436 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1437 ssize_t res; 1438 1439 if (is_bad_inode(inode)) 1440 return -EIO; 1441 1442 /* Don't allow parallel writes to the same file */ 1443 inode_lock(inode); 1444 res = generic_write_checks(iocb, from); 1445 if (res > 0) 1446 res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); 1447 fuse_invalidate_attr(inode); 1448 if (res > 0) 1449 fuse_write_update_size(inode, iocb->ki_pos); 1450 inode_unlock(inode); 1451 1452 return res; 1453 } 1454 1455 static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req) 1456 { 1457 int i; 1458 1459 for (i = 0; i < req->num_pages; i++) 1460 __free_page(req->pages[i]); 1461 1462 if (req->ff) 1463 fuse_file_put(req->ff, false); 1464 } 1465 1466 static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req) 1467 { 1468 struct inode *inode = req->inode; 1469 struct fuse_inode *fi = get_fuse_inode(inode); 1470 struct backing_dev_info *bdi = inode_to_bdi(inode); 1471 int i; 1472 1473 list_del(&req->writepages_entry); 1474 for (i = 0; i < req->num_pages; i++) { 1475 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 1476 dec_node_page_state(req->pages[i], NR_WRITEBACK_TEMP); 1477 wb_writeout_inc(&bdi->wb); 1478 } 1479 wake_up(&fi->page_waitq); 1480 } 1481 1482 /* Called under fc->lock, may release and reacquire it */ 1483 static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req, 1484 loff_t size) 1485 __releases(fc->lock) 1486 __acquires(fc->lock) 1487 { 1488 struct fuse_inode *fi = get_fuse_inode(req->inode); 1489 struct fuse_write_in *inarg = &req->misc.write.in; 1490 __u64 data_size = req->num_pages * PAGE_SIZE; 1491 bool queued; 1492 1493 if (!fc->connected) 1494 goto out_free; 1495 1496 if (inarg->offset + data_size <= size) { 1497 inarg->size = data_size; 1498 } else if (inarg->offset < size) { 1499 inarg->size = size - inarg->offset; 1500 } else { 1501 /* Got truncated off completely */ 1502 goto out_free; 1503 } 1504 1505 req->in.args[1].size = inarg->size; 1506 fi->writectr++; 1507 queued = fuse_request_queue_background(fc, req); 1508 WARN_ON(!queued); 1509 return; 1510 1511 out_free: 1512 fuse_writepage_finish(fc, req); 1513 spin_unlock(&fc->lock); 1514 fuse_writepage_free(fc, req); 1515 fuse_put_request(fc, req); 1516 spin_lock(&fc->lock); 1517 } 1518 1519 /* 1520 * If fi->writectr is positive (no truncate or fsync going on) send 1521 * all queued writepage requests. 1522 * 1523 * Called with fc->lock 1524 */ 1525 void fuse_flush_writepages(struct inode *inode) 1526 __releases(fc->lock) 1527 __acquires(fc->lock) 1528 { 1529 struct fuse_conn *fc = get_fuse_conn(inode); 1530 struct fuse_inode *fi = get_fuse_inode(inode); 1531 size_t crop = i_size_read(inode); 1532 struct fuse_req *req; 1533 1534 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { 1535 req = list_entry(fi->queued_writes.next, struct fuse_req, list); 1536 list_del_init(&req->list); 1537 fuse_send_writepage(fc, req, crop); 1538 } 1539 } 1540 1541 static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req) 1542 { 1543 struct inode *inode = req->inode; 1544 struct fuse_inode *fi = get_fuse_inode(inode); 1545 1546 mapping_set_error(inode->i_mapping, req->out.h.error); 1547 spin_lock(&fc->lock); 1548 while (req->misc.write.next) { 1549 struct fuse_conn *fc = get_fuse_conn(inode); 1550 struct fuse_write_in *inarg = &req->misc.write.in; 1551 struct fuse_req *next = req->misc.write.next; 1552 req->misc.write.next = next->misc.write.next; 1553 next->misc.write.next = NULL; 1554 next->ff = fuse_file_get(req->ff); 1555 list_add(&next->writepages_entry, &fi->writepages); 1556 1557 /* 1558 * Skip fuse_flush_writepages() to make it easy to crop requests 1559 * based on primary request size. 1560 * 1561 * 1st case (trivial): there are no concurrent activities using 1562 * fuse_set/release_nowrite. Then we're on safe side because 1563 * fuse_flush_writepages() would call fuse_send_writepage() 1564 * anyway. 1565 * 1566 * 2nd case: someone called fuse_set_nowrite and it is waiting 1567 * now for completion of all in-flight requests. This happens 1568 * rarely and no more than once per page, so this should be 1569 * okay. 1570 * 1571 * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle 1572 * of fuse_set_nowrite..fuse_release_nowrite section. The fact 1573 * that fuse_set_nowrite returned implies that all in-flight 1574 * requests were completed along with all of their secondary 1575 * requests. Further primary requests are blocked by negative 1576 * writectr. Hence there cannot be any in-flight requests and 1577 * no invocations of fuse_writepage_end() while we're in 1578 * fuse_set_nowrite..fuse_release_nowrite section. 1579 */ 1580 fuse_send_writepage(fc, next, inarg->offset + inarg->size); 1581 } 1582 fi->writectr--; 1583 fuse_writepage_finish(fc, req); 1584 spin_unlock(&fc->lock); 1585 fuse_writepage_free(fc, req); 1586 } 1587 1588 static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc, 1589 struct fuse_inode *fi) 1590 { 1591 struct fuse_file *ff = NULL; 1592 1593 spin_lock(&fc->lock); 1594 if (!list_empty(&fi->write_files)) { 1595 ff = list_entry(fi->write_files.next, struct fuse_file, 1596 write_entry); 1597 fuse_file_get(ff); 1598 } 1599 spin_unlock(&fc->lock); 1600 1601 return ff; 1602 } 1603 1604 static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc, 1605 struct fuse_inode *fi) 1606 { 1607 struct fuse_file *ff = __fuse_write_file_get(fc, fi); 1608 WARN_ON(!ff); 1609 return ff; 1610 } 1611 1612 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) 1613 { 1614 struct fuse_conn *fc = get_fuse_conn(inode); 1615 struct fuse_inode *fi = get_fuse_inode(inode); 1616 struct fuse_file *ff; 1617 int err; 1618 1619 ff = __fuse_write_file_get(fc, fi); 1620 err = fuse_flush_times(inode, ff); 1621 if (ff) 1622 fuse_file_put(ff, 0); 1623 1624 return err; 1625 } 1626 1627 static int fuse_writepage_locked(struct page *page) 1628 { 1629 struct address_space *mapping = page->mapping; 1630 struct inode *inode = mapping->host; 1631 struct fuse_conn *fc = get_fuse_conn(inode); 1632 struct fuse_inode *fi = get_fuse_inode(inode); 1633 struct fuse_req *req; 1634 struct page *tmp_page; 1635 int error = -ENOMEM; 1636 1637 set_page_writeback(page); 1638 1639 req = fuse_request_alloc_nofs(1); 1640 if (!req) 1641 goto err; 1642 1643 /* writeback always goes to bg_queue */ 1644 __set_bit(FR_BACKGROUND, &req->flags); 1645 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 1646 if (!tmp_page) 1647 goto err_free; 1648 1649 error = -EIO; 1650 req->ff = fuse_write_file_get(fc, fi); 1651 if (!req->ff) 1652 goto err_nofile; 1653 1654 fuse_write_fill(req, req->ff, page_offset(page), 0); 1655 1656 copy_highpage(tmp_page, page); 1657 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE; 1658 req->misc.write.next = NULL; 1659 req->in.argpages = 1; 1660 req->num_pages = 1; 1661 req->pages[0] = tmp_page; 1662 req->page_descs[0].offset = 0; 1663 req->page_descs[0].length = PAGE_SIZE; 1664 req->end = fuse_writepage_end; 1665 req->inode = inode; 1666 1667 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 1668 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 1669 1670 spin_lock(&fc->lock); 1671 list_add(&req->writepages_entry, &fi->writepages); 1672 list_add_tail(&req->list, &fi->queued_writes); 1673 fuse_flush_writepages(inode); 1674 spin_unlock(&fc->lock); 1675 1676 end_page_writeback(page); 1677 1678 return 0; 1679 1680 err_nofile: 1681 __free_page(tmp_page); 1682 err_free: 1683 fuse_request_free(req); 1684 err: 1685 mapping_set_error(page->mapping, error); 1686 end_page_writeback(page); 1687 return error; 1688 } 1689 1690 static int fuse_writepage(struct page *page, struct writeback_control *wbc) 1691 { 1692 int err; 1693 1694 if (fuse_page_is_writeback(page->mapping->host, page->index)) { 1695 /* 1696 * ->writepages() should be called for sync() and friends. We 1697 * should only get here on direct reclaim and then we are 1698 * allowed to skip a page which is already in flight 1699 */ 1700 WARN_ON(wbc->sync_mode == WB_SYNC_ALL); 1701 1702 redirty_page_for_writepage(wbc, page); 1703 return 0; 1704 } 1705 1706 err = fuse_writepage_locked(page); 1707 unlock_page(page); 1708 1709 return err; 1710 } 1711 1712 struct fuse_fill_wb_data { 1713 struct fuse_req *req; 1714 struct fuse_file *ff; 1715 struct inode *inode; 1716 struct page **orig_pages; 1717 }; 1718 1719 static void fuse_writepages_send(struct fuse_fill_wb_data *data) 1720 { 1721 struct fuse_req *req = data->req; 1722 struct inode *inode = data->inode; 1723 struct fuse_conn *fc = get_fuse_conn(inode); 1724 struct fuse_inode *fi = get_fuse_inode(inode); 1725 int num_pages = req->num_pages; 1726 int i; 1727 1728 req->ff = fuse_file_get(data->ff); 1729 spin_lock(&fc->lock); 1730 list_add_tail(&req->list, &fi->queued_writes); 1731 fuse_flush_writepages(inode); 1732 spin_unlock(&fc->lock); 1733 1734 for (i = 0; i < num_pages; i++) 1735 end_page_writeback(data->orig_pages[i]); 1736 } 1737 1738 static bool fuse_writepage_in_flight(struct fuse_req *new_req, 1739 struct page *page) 1740 { 1741 struct fuse_conn *fc = get_fuse_conn(new_req->inode); 1742 struct fuse_inode *fi = get_fuse_inode(new_req->inode); 1743 struct fuse_req *tmp; 1744 struct fuse_req *old_req; 1745 bool found = false; 1746 pgoff_t curr_index; 1747 1748 BUG_ON(new_req->num_pages != 0); 1749 1750 spin_lock(&fc->lock); 1751 list_del(&new_req->writepages_entry); 1752 list_for_each_entry(old_req, &fi->writepages, writepages_entry) { 1753 BUG_ON(old_req->inode != new_req->inode); 1754 curr_index = old_req->misc.write.in.offset >> PAGE_SHIFT; 1755 if (curr_index <= page->index && 1756 page->index < curr_index + old_req->num_pages) { 1757 found = true; 1758 break; 1759 } 1760 } 1761 if (!found) { 1762 list_add(&new_req->writepages_entry, &fi->writepages); 1763 goto out_unlock; 1764 } 1765 1766 new_req->num_pages = 1; 1767 for (tmp = old_req; tmp != NULL; tmp = tmp->misc.write.next) { 1768 BUG_ON(tmp->inode != new_req->inode); 1769 curr_index = tmp->misc.write.in.offset >> PAGE_SHIFT; 1770 if (tmp->num_pages == 1 && 1771 curr_index == page->index) { 1772 old_req = tmp; 1773 } 1774 } 1775 1776 if (old_req->num_pages == 1 && test_bit(FR_PENDING, &old_req->flags)) { 1777 struct backing_dev_info *bdi = inode_to_bdi(page->mapping->host); 1778 1779 copy_highpage(old_req->pages[0], page); 1780 spin_unlock(&fc->lock); 1781 1782 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 1783 dec_node_page_state(page, NR_WRITEBACK_TEMP); 1784 wb_writeout_inc(&bdi->wb); 1785 fuse_writepage_free(fc, new_req); 1786 fuse_request_free(new_req); 1787 goto out; 1788 } else { 1789 new_req->misc.write.next = old_req->misc.write.next; 1790 old_req->misc.write.next = new_req; 1791 } 1792 out_unlock: 1793 spin_unlock(&fc->lock); 1794 out: 1795 return found; 1796 } 1797 1798 static int fuse_writepages_fill(struct page *page, 1799 struct writeback_control *wbc, void *_data) 1800 { 1801 struct fuse_fill_wb_data *data = _data; 1802 struct fuse_req *req = data->req; 1803 struct inode *inode = data->inode; 1804 struct fuse_conn *fc = get_fuse_conn(inode); 1805 struct page *tmp_page; 1806 bool is_writeback; 1807 int err; 1808 1809 if (!data->ff) { 1810 err = -EIO; 1811 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode)); 1812 if (!data->ff) 1813 goto out_unlock; 1814 } 1815 1816 /* 1817 * Being under writeback is unlikely but possible. For example direct 1818 * read to an mmaped fuse file will set the page dirty twice; once when 1819 * the pages are faulted with get_user_pages(), and then after the read 1820 * completed. 1821 */ 1822 is_writeback = fuse_page_is_writeback(inode, page->index); 1823 1824 if (req && req->num_pages && 1825 (is_writeback || req->num_pages == fc->max_pages || 1826 (req->num_pages + 1) * PAGE_SIZE > fc->max_write || 1827 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) { 1828 fuse_writepages_send(data); 1829 data->req = NULL; 1830 } else if (req && req->num_pages == req->max_pages) { 1831 if (!fuse_req_realloc_pages(fc, req, GFP_NOFS)) { 1832 fuse_writepages_send(data); 1833 req = data->req = NULL; 1834 } 1835 } 1836 1837 err = -ENOMEM; 1838 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 1839 if (!tmp_page) 1840 goto out_unlock; 1841 1842 /* 1843 * The page must not be redirtied until the writeout is completed 1844 * (i.e. userspace has sent a reply to the write request). Otherwise 1845 * there could be more than one temporary page instance for each real 1846 * page. 1847 * 1848 * This is ensured by holding the page lock in page_mkwrite() while 1849 * checking fuse_page_is_writeback(). We already hold the page lock 1850 * since clear_page_dirty_for_io() and keep it held until we add the 1851 * request to the fi->writepages list and increment req->num_pages. 1852 * After this fuse_page_is_writeback() will indicate that the page is 1853 * under writeback, so we can release the page lock. 1854 */ 1855 if (data->req == NULL) { 1856 struct fuse_inode *fi = get_fuse_inode(inode); 1857 1858 err = -ENOMEM; 1859 req = fuse_request_alloc_nofs(FUSE_REQ_INLINE_PAGES); 1860 if (!req) { 1861 __free_page(tmp_page); 1862 goto out_unlock; 1863 } 1864 1865 fuse_write_fill(req, data->ff, page_offset(page), 0); 1866 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE; 1867 req->misc.write.next = NULL; 1868 req->in.argpages = 1; 1869 __set_bit(FR_BACKGROUND, &req->flags); 1870 req->num_pages = 0; 1871 req->end = fuse_writepage_end; 1872 req->inode = inode; 1873 1874 spin_lock(&fc->lock); 1875 list_add(&req->writepages_entry, &fi->writepages); 1876 spin_unlock(&fc->lock); 1877 1878 data->req = req; 1879 } 1880 set_page_writeback(page); 1881 1882 copy_highpage(tmp_page, page); 1883 req->pages[req->num_pages] = tmp_page; 1884 req->page_descs[req->num_pages].offset = 0; 1885 req->page_descs[req->num_pages].length = PAGE_SIZE; 1886 1887 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 1888 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 1889 1890 err = 0; 1891 if (is_writeback && fuse_writepage_in_flight(req, page)) { 1892 end_page_writeback(page); 1893 data->req = NULL; 1894 goto out_unlock; 1895 } 1896 data->orig_pages[req->num_pages] = page; 1897 1898 /* 1899 * Protected by fc->lock against concurrent access by 1900 * fuse_page_is_writeback(). 1901 */ 1902 spin_lock(&fc->lock); 1903 req->num_pages++; 1904 spin_unlock(&fc->lock); 1905 1906 out_unlock: 1907 unlock_page(page); 1908 1909 return err; 1910 } 1911 1912 static int fuse_writepages(struct address_space *mapping, 1913 struct writeback_control *wbc) 1914 { 1915 struct inode *inode = mapping->host; 1916 struct fuse_conn *fc = get_fuse_conn(inode); 1917 struct fuse_fill_wb_data data; 1918 int err; 1919 1920 err = -EIO; 1921 if (is_bad_inode(inode)) 1922 goto out; 1923 1924 data.inode = inode; 1925 data.req = NULL; 1926 data.ff = NULL; 1927 1928 err = -ENOMEM; 1929 data.orig_pages = kcalloc(fc->max_pages, 1930 sizeof(struct page *), 1931 GFP_NOFS); 1932 if (!data.orig_pages) 1933 goto out; 1934 1935 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); 1936 if (data.req) { 1937 /* Ignore errors if we can write at least one page */ 1938 BUG_ON(!data.req->num_pages); 1939 fuse_writepages_send(&data); 1940 err = 0; 1941 } 1942 if (data.ff) 1943 fuse_file_put(data.ff, false); 1944 1945 kfree(data.orig_pages); 1946 out: 1947 return err; 1948 } 1949 1950 /* 1951 * It's worthy to make sure that space is reserved on disk for the write, 1952 * but how to implement it without killing performance need more thinking. 1953 */ 1954 static int fuse_write_begin(struct file *file, struct address_space *mapping, 1955 loff_t pos, unsigned len, unsigned flags, 1956 struct page **pagep, void **fsdata) 1957 { 1958 pgoff_t index = pos >> PAGE_SHIFT; 1959 struct fuse_conn *fc = get_fuse_conn(file_inode(file)); 1960 struct page *page; 1961 loff_t fsize; 1962 int err = -ENOMEM; 1963 1964 WARN_ON(!fc->writeback_cache); 1965 1966 page = grab_cache_page_write_begin(mapping, index, flags); 1967 if (!page) 1968 goto error; 1969 1970 fuse_wait_on_page_writeback(mapping->host, page->index); 1971 1972 if (PageUptodate(page) || len == PAGE_SIZE) 1973 goto success; 1974 /* 1975 * Check if the start this page comes after the end of file, in which 1976 * case the readpage can be optimized away. 1977 */ 1978 fsize = i_size_read(mapping->host); 1979 if (fsize <= (pos & PAGE_MASK)) { 1980 size_t off = pos & ~PAGE_MASK; 1981 if (off) 1982 zero_user_segment(page, 0, off); 1983 goto success; 1984 } 1985 err = fuse_do_readpage(file, page); 1986 if (err) 1987 goto cleanup; 1988 success: 1989 *pagep = page; 1990 return 0; 1991 1992 cleanup: 1993 unlock_page(page); 1994 put_page(page); 1995 error: 1996 return err; 1997 } 1998 1999 static int fuse_write_end(struct file *file, struct address_space *mapping, 2000 loff_t pos, unsigned len, unsigned copied, 2001 struct page *page, void *fsdata) 2002 { 2003 struct inode *inode = page->mapping->host; 2004 2005 /* Haven't copied anything? Skip zeroing, size extending, dirtying. */ 2006 if (!copied) 2007 goto unlock; 2008 2009 if (!PageUptodate(page)) { 2010 /* Zero any unwritten bytes at the end of the page */ 2011 size_t endoff = (pos + copied) & ~PAGE_MASK; 2012 if (endoff) 2013 zero_user_segment(page, endoff, PAGE_SIZE); 2014 SetPageUptodate(page); 2015 } 2016 2017 fuse_write_update_size(inode, pos + copied); 2018 set_page_dirty(page); 2019 2020 unlock: 2021 unlock_page(page); 2022 put_page(page); 2023 2024 return copied; 2025 } 2026 2027 static int fuse_launder_page(struct page *page) 2028 { 2029 int err = 0; 2030 if (clear_page_dirty_for_io(page)) { 2031 struct inode *inode = page->mapping->host; 2032 err = fuse_writepage_locked(page); 2033 if (!err) 2034 fuse_wait_on_page_writeback(inode, page->index); 2035 } 2036 return err; 2037 } 2038 2039 /* 2040 * Write back dirty pages now, because there may not be any suitable 2041 * open files later 2042 */ 2043 static void fuse_vma_close(struct vm_area_struct *vma) 2044 { 2045 filemap_write_and_wait(vma->vm_file->f_mapping); 2046 } 2047 2048 /* 2049 * Wait for writeback against this page to complete before allowing it 2050 * to be marked dirty again, and hence written back again, possibly 2051 * before the previous writepage completed. 2052 * 2053 * Block here, instead of in ->writepage(), so that the userspace fs 2054 * can only block processes actually operating on the filesystem. 2055 * 2056 * Otherwise unprivileged userspace fs would be able to block 2057 * unrelated: 2058 * 2059 * - page migration 2060 * - sync(2) 2061 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER 2062 */ 2063 static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) 2064 { 2065 struct page *page = vmf->page; 2066 struct inode *inode = file_inode(vmf->vma->vm_file); 2067 2068 file_update_time(vmf->vma->vm_file); 2069 lock_page(page); 2070 if (page->mapping != inode->i_mapping) { 2071 unlock_page(page); 2072 return VM_FAULT_NOPAGE; 2073 } 2074 2075 fuse_wait_on_page_writeback(inode, page->index); 2076 return VM_FAULT_LOCKED; 2077 } 2078 2079 static const struct vm_operations_struct fuse_file_vm_ops = { 2080 .close = fuse_vma_close, 2081 .fault = filemap_fault, 2082 .map_pages = filemap_map_pages, 2083 .page_mkwrite = fuse_page_mkwrite, 2084 }; 2085 2086 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) 2087 { 2088 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 2089 fuse_link_write_file(file); 2090 2091 file_accessed(file); 2092 vma->vm_ops = &fuse_file_vm_ops; 2093 return 0; 2094 } 2095 2096 static int fuse_direct_mmap(struct file *file, struct vm_area_struct *vma) 2097 { 2098 /* Can't provide the coherency needed for MAP_SHARED */ 2099 if (vma->vm_flags & VM_MAYSHARE) 2100 return -ENODEV; 2101 2102 invalidate_inode_pages2(file->f_mapping); 2103 2104 return generic_file_mmap(file, vma); 2105 } 2106 2107 static int convert_fuse_file_lock(struct fuse_conn *fc, 2108 const struct fuse_file_lock *ffl, 2109 struct file_lock *fl) 2110 { 2111 switch (ffl->type) { 2112 case F_UNLCK: 2113 break; 2114 2115 case F_RDLCK: 2116 case F_WRLCK: 2117 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX || 2118 ffl->end < ffl->start) 2119 return -EIO; 2120 2121 fl->fl_start = ffl->start; 2122 fl->fl_end = ffl->end; 2123 2124 /* 2125 * Convert pid into init's pid namespace. The locks API will 2126 * translate it into the caller's pid namespace. 2127 */ 2128 rcu_read_lock(); 2129 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns); 2130 rcu_read_unlock(); 2131 break; 2132 2133 default: 2134 return -EIO; 2135 } 2136 fl->fl_type = ffl->type; 2137 return 0; 2138 } 2139 2140 static void fuse_lk_fill(struct fuse_args *args, struct file *file, 2141 const struct file_lock *fl, int opcode, pid_t pid, 2142 int flock, struct fuse_lk_in *inarg) 2143 { 2144 struct inode *inode = file_inode(file); 2145 struct fuse_conn *fc = get_fuse_conn(inode); 2146 struct fuse_file *ff = file->private_data; 2147 2148 memset(inarg, 0, sizeof(*inarg)); 2149 inarg->fh = ff->fh; 2150 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner); 2151 inarg->lk.start = fl->fl_start; 2152 inarg->lk.end = fl->fl_end; 2153 inarg->lk.type = fl->fl_type; 2154 inarg->lk.pid = pid; 2155 if (flock) 2156 inarg->lk_flags |= FUSE_LK_FLOCK; 2157 args->in.h.opcode = opcode; 2158 args->in.h.nodeid = get_node_id(inode); 2159 args->in.numargs = 1; 2160 args->in.args[0].size = sizeof(*inarg); 2161 args->in.args[0].value = inarg; 2162 } 2163 2164 static int fuse_getlk(struct file *file, struct file_lock *fl) 2165 { 2166 struct inode *inode = file_inode(file); 2167 struct fuse_conn *fc = get_fuse_conn(inode); 2168 FUSE_ARGS(args); 2169 struct fuse_lk_in inarg; 2170 struct fuse_lk_out outarg; 2171 int err; 2172 2173 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg); 2174 args.out.numargs = 1; 2175 args.out.args[0].size = sizeof(outarg); 2176 args.out.args[0].value = &outarg; 2177 err = fuse_simple_request(fc, &args); 2178 if (!err) 2179 err = convert_fuse_file_lock(fc, &outarg.lk, fl); 2180 2181 return err; 2182 } 2183 2184 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) 2185 { 2186 struct inode *inode = file_inode(file); 2187 struct fuse_conn *fc = get_fuse_conn(inode); 2188 FUSE_ARGS(args); 2189 struct fuse_lk_in inarg; 2190 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; 2191 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL; 2192 pid_t pid_nr = pid_nr_ns(pid, fc->pid_ns); 2193 int err; 2194 2195 if (fl->fl_lmops && fl->fl_lmops->lm_grant) { 2196 /* NLM needs asynchronous locks, which we don't support yet */ 2197 return -ENOLCK; 2198 } 2199 2200 /* Unlock on close is handled by the flush method */ 2201 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX) 2202 return 0; 2203 2204 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); 2205 err = fuse_simple_request(fc, &args); 2206 2207 /* locking is restartable */ 2208 if (err == -EINTR) 2209 err = -ERESTARTSYS; 2210 2211 return err; 2212 } 2213 2214 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) 2215 { 2216 struct inode *inode = file_inode(file); 2217 struct fuse_conn *fc = get_fuse_conn(inode); 2218 int err; 2219 2220 if (cmd == F_CANCELLK) { 2221 err = 0; 2222 } else if (cmd == F_GETLK) { 2223 if (fc->no_lock) { 2224 posix_test_lock(file, fl); 2225 err = 0; 2226 } else 2227 err = fuse_getlk(file, fl); 2228 } else { 2229 if (fc->no_lock) 2230 err = posix_lock_file(file, fl, NULL); 2231 else 2232 err = fuse_setlk(file, fl, 0); 2233 } 2234 return err; 2235 } 2236 2237 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) 2238 { 2239 struct inode *inode = file_inode(file); 2240 struct fuse_conn *fc = get_fuse_conn(inode); 2241 int err; 2242 2243 if (fc->no_flock) { 2244 err = locks_lock_file_wait(file, fl); 2245 } else { 2246 struct fuse_file *ff = file->private_data; 2247 2248 /* emulate flock with POSIX locks */ 2249 ff->flock = true; 2250 err = fuse_setlk(file, fl, 1); 2251 } 2252 2253 return err; 2254 } 2255 2256 static sector_t fuse_bmap(struct address_space *mapping, sector_t block) 2257 { 2258 struct inode *inode = mapping->host; 2259 struct fuse_conn *fc = get_fuse_conn(inode); 2260 FUSE_ARGS(args); 2261 struct fuse_bmap_in inarg; 2262 struct fuse_bmap_out outarg; 2263 int err; 2264 2265 if (!inode->i_sb->s_bdev || fc->no_bmap) 2266 return 0; 2267 2268 memset(&inarg, 0, sizeof(inarg)); 2269 inarg.block = block; 2270 inarg.blocksize = inode->i_sb->s_blocksize; 2271 args.in.h.opcode = FUSE_BMAP; 2272 args.in.h.nodeid = get_node_id(inode); 2273 args.in.numargs = 1; 2274 args.in.args[0].size = sizeof(inarg); 2275 args.in.args[0].value = &inarg; 2276 args.out.numargs = 1; 2277 args.out.args[0].size = sizeof(outarg); 2278 args.out.args[0].value = &outarg; 2279 err = fuse_simple_request(fc, &args); 2280 if (err == -ENOSYS) 2281 fc->no_bmap = 1; 2282 2283 return err ? 0 : outarg.block; 2284 } 2285 2286 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) 2287 { 2288 struct inode *inode = file->f_mapping->host; 2289 struct fuse_conn *fc = get_fuse_conn(inode); 2290 struct fuse_file *ff = file->private_data; 2291 FUSE_ARGS(args); 2292 struct fuse_lseek_in inarg = { 2293 .fh = ff->fh, 2294 .offset = offset, 2295 .whence = whence 2296 }; 2297 struct fuse_lseek_out outarg; 2298 int err; 2299 2300 if (fc->no_lseek) 2301 goto fallback; 2302 2303 args.in.h.opcode = FUSE_LSEEK; 2304 args.in.h.nodeid = ff->nodeid; 2305 args.in.numargs = 1; 2306 args.in.args[0].size = sizeof(inarg); 2307 args.in.args[0].value = &inarg; 2308 args.out.numargs = 1; 2309 args.out.args[0].size = sizeof(outarg); 2310 args.out.args[0].value = &outarg; 2311 err = fuse_simple_request(fc, &args); 2312 if (err) { 2313 if (err == -ENOSYS) { 2314 fc->no_lseek = 1; 2315 goto fallback; 2316 } 2317 return err; 2318 } 2319 2320 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes); 2321 2322 fallback: 2323 err = fuse_update_attributes(inode, file); 2324 if (!err) 2325 return generic_file_llseek(file, offset, whence); 2326 else 2327 return err; 2328 } 2329 2330 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) 2331 { 2332 loff_t retval; 2333 struct inode *inode = file_inode(file); 2334 2335 switch (whence) { 2336 case SEEK_SET: 2337 case SEEK_CUR: 2338 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */ 2339 retval = generic_file_llseek(file, offset, whence); 2340 break; 2341 case SEEK_END: 2342 inode_lock(inode); 2343 retval = fuse_update_attributes(inode, file); 2344 if (!retval) 2345 retval = generic_file_llseek(file, offset, whence); 2346 inode_unlock(inode); 2347 break; 2348 case SEEK_HOLE: 2349 case SEEK_DATA: 2350 inode_lock(inode); 2351 retval = fuse_lseek(file, offset, whence); 2352 inode_unlock(inode); 2353 break; 2354 default: 2355 retval = -EINVAL; 2356 } 2357 2358 return retval; 2359 } 2360 2361 /* 2362 * CUSE servers compiled on 32bit broke on 64bit kernels because the 2363 * ABI was defined to be 'struct iovec' which is different on 32bit 2364 * and 64bit. Fortunately we can determine which structure the server 2365 * used from the size of the reply. 2366 */ 2367 static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src, 2368 size_t transferred, unsigned count, 2369 bool is_compat) 2370 { 2371 #ifdef CONFIG_COMPAT 2372 if (count * sizeof(struct compat_iovec) == transferred) { 2373 struct compat_iovec *ciov = src; 2374 unsigned i; 2375 2376 /* 2377 * With this interface a 32bit server cannot support 2378 * non-compat (i.e. ones coming from 64bit apps) ioctl 2379 * requests 2380 */ 2381 if (!is_compat) 2382 return -EINVAL; 2383 2384 for (i = 0; i < count; i++) { 2385 dst[i].iov_base = compat_ptr(ciov[i].iov_base); 2386 dst[i].iov_len = ciov[i].iov_len; 2387 } 2388 return 0; 2389 } 2390 #endif 2391 2392 if (count * sizeof(struct iovec) != transferred) 2393 return -EIO; 2394 2395 memcpy(dst, src, transferred); 2396 return 0; 2397 } 2398 2399 /* Make sure iov_length() won't overflow */ 2400 static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov, 2401 size_t count) 2402 { 2403 size_t n; 2404 u32 max = fc->max_pages << PAGE_SHIFT; 2405 2406 for (n = 0; n < count; n++, iov++) { 2407 if (iov->iov_len > (size_t) max) 2408 return -ENOMEM; 2409 max -= iov->iov_len; 2410 } 2411 return 0; 2412 } 2413 2414 static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst, 2415 void *src, size_t transferred, unsigned count, 2416 bool is_compat) 2417 { 2418 unsigned i; 2419 struct fuse_ioctl_iovec *fiov = src; 2420 2421 if (fc->minor < 16) { 2422 return fuse_copy_ioctl_iovec_old(dst, src, transferred, 2423 count, is_compat); 2424 } 2425 2426 if (count * sizeof(struct fuse_ioctl_iovec) != transferred) 2427 return -EIO; 2428 2429 for (i = 0; i < count; i++) { 2430 /* Did the server supply an inappropriate value? */ 2431 if (fiov[i].base != (unsigned long) fiov[i].base || 2432 fiov[i].len != (unsigned long) fiov[i].len) 2433 return -EIO; 2434 2435 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base; 2436 dst[i].iov_len = (size_t) fiov[i].len; 2437 2438 #ifdef CONFIG_COMPAT 2439 if (is_compat && 2440 (ptr_to_compat(dst[i].iov_base) != fiov[i].base || 2441 (compat_size_t) dst[i].iov_len != fiov[i].len)) 2442 return -EIO; 2443 #endif 2444 } 2445 2446 return 0; 2447 } 2448 2449 2450 /* 2451 * For ioctls, there is no generic way to determine how much memory 2452 * needs to be read and/or written. Furthermore, ioctls are allowed 2453 * to dereference the passed pointer, so the parameter requires deep 2454 * copying but FUSE has no idea whatsoever about what to copy in or 2455 * out. 2456 * 2457 * This is solved by allowing FUSE server to retry ioctl with 2458 * necessary in/out iovecs. Let's assume the ioctl implementation 2459 * needs to read in the following structure. 2460 * 2461 * struct a { 2462 * char *buf; 2463 * size_t buflen; 2464 * } 2465 * 2466 * On the first callout to FUSE server, inarg->in_size and 2467 * inarg->out_size will be NULL; then, the server completes the ioctl 2468 * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and 2469 * the actual iov array to 2470 * 2471 * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) } } 2472 * 2473 * which tells FUSE to copy in the requested area and retry the ioctl. 2474 * On the second round, the server has access to the structure and 2475 * from that it can tell what to look for next, so on the invocation, 2476 * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to 2477 * 2478 * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) }, 2479 * { .iov_base = a.buf, .iov_len = a.buflen } } 2480 * 2481 * FUSE will copy both struct a and the pointed buffer from the 2482 * process doing the ioctl and retry ioctl with both struct a and the 2483 * buffer. 2484 * 2485 * This time, FUSE server has everything it needs and completes ioctl 2486 * without FUSE_IOCTL_RETRY which finishes the ioctl call. 2487 * 2488 * Copying data out works the same way. 2489 * 2490 * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel 2491 * automatically initializes in and out iovs by decoding @cmd with 2492 * _IOC_* macros and the server is not allowed to request RETRY. This 2493 * limits ioctl data transfers to well-formed ioctls and is the forced 2494 * behavior for all FUSE servers. 2495 */ 2496 long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, 2497 unsigned int flags) 2498 { 2499 struct fuse_file *ff = file->private_data; 2500 struct fuse_conn *fc = ff->fc; 2501 struct fuse_ioctl_in inarg = { 2502 .fh = ff->fh, 2503 .cmd = cmd, 2504 .arg = arg, 2505 .flags = flags 2506 }; 2507 struct fuse_ioctl_out outarg; 2508 struct fuse_req *req = NULL; 2509 struct page **pages = NULL; 2510 struct iovec *iov_page = NULL; 2511 struct iovec *in_iov = NULL, *out_iov = NULL; 2512 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages; 2513 size_t in_size, out_size, transferred, c; 2514 int err, i; 2515 struct iov_iter ii; 2516 2517 #if BITS_PER_LONG == 32 2518 inarg.flags |= FUSE_IOCTL_32BIT; 2519 #else 2520 if (flags & FUSE_IOCTL_COMPAT) 2521 inarg.flags |= FUSE_IOCTL_32BIT; 2522 #endif 2523 2524 /* assume all the iovs returned by client always fits in a page */ 2525 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); 2526 2527 err = -ENOMEM; 2528 pages = kcalloc(fc->max_pages, sizeof(pages[0]), GFP_KERNEL); 2529 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); 2530 if (!pages || !iov_page) 2531 goto out; 2532 2533 /* 2534 * If restricted, initialize IO parameters as encoded in @cmd. 2535 * RETRY from server is not allowed. 2536 */ 2537 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) { 2538 struct iovec *iov = iov_page; 2539 2540 iov->iov_base = (void __user *)arg; 2541 iov->iov_len = _IOC_SIZE(cmd); 2542 2543 if (_IOC_DIR(cmd) & _IOC_WRITE) { 2544 in_iov = iov; 2545 in_iovs = 1; 2546 } 2547 2548 if (_IOC_DIR(cmd) & _IOC_READ) { 2549 out_iov = iov; 2550 out_iovs = 1; 2551 } 2552 } 2553 2554 retry: 2555 inarg.in_size = in_size = iov_length(in_iov, in_iovs); 2556 inarg.out_size = out_size = iov_length(out_iov, out_iovs); 2557 2558 /* 2559 * Out data can be used either for actual out data or iovs, 2560 * make sure there always is at least one page. 2561 */ 2562 out_size = max_t(size_t, out_size, PAGE_SIZE); 2563 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE); 2564 2565 /* make sure there are enough buffer pages and init request with them */ 2566 err = -ENOMEM; 2567 if (max_pages > fc->max_pages) 2568 goto out; 2569 while (num_pages < max_pages) { 2570 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); 2571 if (!pages[num_pages]) 2572 goto out; 2573 num_pages++; 2574 } 2575 2576 req = fuse_get_req(fc, num_pages); 2577 if (IS_ERR(req)) { 2578 err = PTR_ERR(req); 2579 req = NULL; 2580 goto out; 2581 } 2582 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages); 2583 req->num_pages = num_pages; 2584 fuse_page_descs_length_init(req, 0, req->num_pages); 2585 2586 /* okay, let's send it to the client */ 2587 req->in.h.opcode = FUSE_IOCTL; 2588 req->in.h.nodeid = ff->nodeid; 2589 req->in.numargs = 1; 2590 req->in.args[0].size = sizeof(inarg); 2591 req->in.args[0].value = &inarg; 2592 if (in_size) { 2593 req->in.numargs++; 2594 req->in.args[1].size = in_size; 2595 req->in.argpages = 1; 2596 2597 err = -EFAULT; 2598 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size); 2599 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) { 2600 c = copy_page_from_iter(pages[i], 0, PAGE_SIZE, &ii); 2601 if (c != PAGE_SIZE && iov_iter_count(&ii)) 2602 goto out; 2603 } 2604 } 2605 2606 req->out.numargs = 2; 2607 req->out.args[0].size = sizeof(outarg); 2608 req->out.args[0].value = &outarg; 2609 req->out.args[1].size = out_size; 2610 req->out.argpages = 1; 2611 req->out.argvar = 1; 2612 2613 fuse_request_send(fc, req); 2614 err = req->out.h.error; 2615 transferred = req->out.args[1].size; 2616 fuse_put_request(fc, req); 2617 req = NULL; 2618 if (err) 2619 goto out; 2620 2621 /* did it ask for retry? */ 2622 if (outarg.flags & FUSE_IOCTL_RETRY) { 2623 void *vaddr; 2624 2625 /* no retry if in restricted mode */ 2626 err = -EIO; 2627 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) 2628 goto out; 2629 2630 in_iovs = outarg.in_iovs; 2631 out_iovs = outarg.out_iovs; 2632 2633 /* 2634 * Make sure things are in boundary, separate checks 2635 * are to protect against overflow. 2636 */ 2637 err = -ENOMEM; 2638 if (in_iovs > FUSE_IOCTL_MAX_IOV || 2639 out_iovs > FUSE_IOCTL_MAX_IOV || 2640 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) 2641 goto out; 2642 2643 vaddr = kmap_atomic(pages[0]); 2644 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr, 2645 transferred, in_iovs + out_iovs, 2646 (flags & FUSE_IOCTL_COMPAT) != 0); 2647 kunmap_atomic(vaddr); 2648 if (err) 2649 goto out; 2650 2651 in_iov = iov_page; 2652 out_iov = in_iov + in_iovs; 2653 2654 err = fuse_verify_ioctl_iov(fc, in_iov, in_iovs); 2655 if (err) 2656 goto out; 2657 2658 err = fuse_verify_ioctl_iov(fc, out_iov, out_iovs); 2659 if (err) 2660 goto out; 2661 2662 goto retry; 2663 } 2664 2665 err = -EIO; 2666 if (transferred > inarg.out_size) 2667 goto out; 2668 2669 err = -EFAULT; 2670 iov_iter_init(&ii, READ, out_iov, out_iovs, transferred); 2671 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) { 2672 c = copy_page_to_iter(pages[i], 0, PAGE_SIZE, &ii); 2673 if (c != PAGE_SIZE && iov_iter_count(&ii)) 2674 goto out; 2675 } 2676 err = 0; 2677 out: 2678 if (req) 2679 fuse_put_request(fc, req); 2680 free_page((unsigned long) iov_page); 2681 while (num_pages) 2682 __free_page(pages[--num_pages]); 2683 kfree(pages); 2684 2685 return err ? err : outarg.result; 2686 } 2687 EXPORT_SYMBOL_GPL(fuse_do_ioctl); 2688 2689 long fuse_ioctl_common(struct file *file, unsigned int cmd, 2690 unsigned long arg, unsigned int flags) 2691 { 2692 struct inode *inode = file_inode(file); 2693 struct fuse_conn *fc = get_fuse_conn(inode); 2694 2695 if (!fuse_allow_current_process(fc)) 2696 return -EACCES; 2697 2698 if (is_bad_inode(inode)) 2699 return -EIO; 2700 2701 return fuse_do_ioctl(file, cmd, arg, flags); 2702 } 2703 2704 static long fuse_file_ioctl(struct file *file, unsigned int cmd, 2705 unsigned long arg) 2706 { 2707 return fuse_ioctl_common(file, cmd, arg, 0); 2708 } 2709 2710 static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd, 2711 unsigned long arg) 2712 { 2713 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT); 2714 } 2715 2716 /* 2717 * All files which have been polled are linked to RB tree 2718 * fuse_conn->polled_files which is indexed by kh. Walk the tree and 2719 * find the matching one. 2720 */ 2721 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, 2722 struct rb_node **parent_out) 2723 { 2724 struct rb_node **link = &fc->polled_files.rb_node; 2725 struct rb_node *last = NULL; 2726 2727 while (*link) { 2728 struct fuse_file *ff; 2729 2730 last = *link; 2731 ff = rb_entry(last, struct fuse_file, polled_node); 2732 2733 if (kh < ff->kh) 2734 link = &last->rb_left; 2735 else if (kh > ff->kh) 2736 link = &last->rb_right; 2737 else 2738 return link; 2739 } 2740 2741 if (parent_out) 2742 *parent_out = last; 2743 return link; 2744 } 2745 2746 /* 2747 * The file is about to be polled. Make sure it's on the polled_files 2748 * RB tree. Note that files once added to the polled_files tree are 2749 * not removed before the file is released. This is because a file 2750 * polled once is likely to be polled again. 2751 */ 2752 static void fuse_register_polled_file(struct fuse_conn *fc, 2753 struct fuse_file *ff) 2754 { 2755 spin_lock(&fc->lock); 2756 if (RB_EMPTY_NODE(&ff->polled_node)) { 2757 struct rb_node **link, *uninitialized_var(parent); 2758 2759 link = fuse_find_polled_node(fc, ff->kh, &parent); 2760 BUG_ON(*link); 2761 rb_link_node(&ff->polled_node, parent, link); 2762 rb_insert_color(&ff->polled_node, &fc->polled_files); 2763 } 2764 spin_unlock(&fc->lock); 2765 } 2766 2767 __poll_t fuse_file_poll(struct file *file, poll_table *wait) 2768 { 2769 struct fuse_file *ff = file->private_data; 2770 struct fuse_conn *fc = ff->fc; 2771 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; 2772 struct fuse_poll_out outarg; 2773 FUSE_ARGS(args); 2774 int err; 2775 2776 if (fc->no_poll) 2777 return DEFAULT_POLLMASK; 2778 2779 poll_wait(file, &ff->poll_wait, wait); 2780 inarg.events = mangle_poll(poll_requested_events(wait)); 2781 2782 /* 2783 * Ask for notification iff there's someone waiting for it. 2784 * The client may ignore the flag and always notify. 2785 */ 2786 if (waitqueue_active(&ff->poll_wait)) { 2787 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; 2788 fuse_register_polled_file(fc, ff); 2789 } 2790 2791 args.in.h.opcode = FUSE_POLL; 2792 args.in.h.nodeid = ff->nodeid; 2793 args.in.numargs = 1; 2794 args.in.args[0].size = sizeof(inarg); 2795 args.in.args[0].value = &inarg; 2796 args.out.numargs = 1; 2797 args.out.args[0].size = sizeof(outarg); 2798 args.out.args[0].value = &outarg; 2799 err = fuse_simple_request(fc, &args); 2800 2801 if (!err) 2802 return demangle_poll(outarg.revents); 2803 if (err == -ENOSYS) { 2804 fc->no_poll = 1; 2805 return DEFAULT_POLLMASK; 2806 } 2807 return EPOLLERR; 2808 } 2809 EXPORT_SYMBOL_GPL(fuse_file_poll); 2810 2811 /* 2812 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and 2813 * wakes up the poll waiters. 2814 */ 2815 int fuse_notify_poll_wakeup(struct fuse_conn *fc, 2816 struct fuse_notify_poll_wakeup_out *outarg) 2817 { 2818 u64 kh = outarg->kh; 2819 struct rb_node **link; 2820 2821 spin_lock(&fc->lock); 2822 2823 link = fuse_find_polled_node(fc, kh, NULL); 2824 if (*link) { 2825 struct fuse_file *ff; 2826 2827 ff = rb_entry(*link, struct fuse_file, polled_node); 2828 wake_up_interruptible_sync(&ff->poll_wait); 2829 } 2830 2831 spin_unlock(&fc->lock); 2832 return 0; 2833 } 2834 2835 static void fuse_do_truncate(struct file *file) 2836 { 2837 struct inode *inode = file->f_mapping->host; 2838 struct iattr attr; 2839 2840 attr.ia_valid = ATTR_SIZE; 2841 attr.ia_size = i_size_read(inode); 2842 2843 attr.ia_file = file; 2844 attr.ia_valid |= ATTR_FILE; 2845 2846 fuse_do_setattr(file_dentry(file), &attr, file); 2847 } 2848 2849 static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) 2850 { 2851 return round_up(off, fc->max_pages << PAGE_SHIFT); 2852 } 2853 2854 static ssize_t 2855 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 2856 { 2857 DECLARE_COMPLETION_ONSTACK(wait); 2858 ssize_t ret = 0; 2859 struct file *file = iocb->ki_filp; 2860 struct fuse_file *ff = file->private_data; 2861 bool async_dio = ff->fc->async_dio; 2862 loff_t pos = 0; 2863 struct inode *inode; 2864 loff_t i_size; 2865 size_t count = iov_iter_count(iter); 2866 loff_t offset = iocb->ki_pos; 2867 struct fuse_io_priv *io; 2868 2869 pos = offset; 2870 inode = file->f_mapping->host; 2871 i_size = i_size_read(inode); 2872 2873 if ((iov_iter_rw(iter) == READ) && (offset > i_size)) 2874 return 0; 2875 2876 /* optimization for short read */ 2877 if (async_dio && iov_iter_rw(iter) != WRITE && offset + count > i_size) { 2878 if (offset >= i_size) 2879 return 0; 2880 iov_iter_truncate(iter, fuse_round_up(ff->fc, i_size - offset)); 2881 count = iov_iter_count(iter); 2882 } 2883 2884 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL); 2885 if (!io) 2886 return -ENOMEM; 2887 spin_lock_init(&io->lock); 2888 kref_init(&io->refcnt); 2889 io->reqs = 1; 2890 io->bytes = -1; 2891 io->size = 0; 2892 io->offset = offset; 2893 io->write = (iov_iter_rw(iter) == WRITE); 2894 io->err = 0; 2895 /* 2896 * By default, we want to optimize all I/Os with async request 2897 * submission to the client filesystem if supported. 2898 */ 2899 io->async = async_dio; 2900 io->iocb = iocb; 2901 io->blocking = is_sync_kiocb(iocb); 2902 2903 /* 2904 * We cannot asynchronously extend the size of a file. 2905 * In such case the aio will behave exactly like sync io. 2906 */ 2907 if ((offset + count > i_size) && iov_iter_rw(iter) == WRITE) 2908 io->blocking = true; 2909 2910 if (io->async && io->blocking) { 2911 /* 2912 * Additional reference to keep io around after 2913 * calling fuse_aio_complete() 2914 */ 2915 kref_get(&io->refcnt); 2916 io->done = &wait; 2917 } 2918 2919 if (iov_iter_rw(iter) == WRITE) { 2920 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); 2921 fuse_invalidate_attr(inode); 2922 } else { 2923 ret = __fuse_direct_read(io, iter, &pos); 2924 } 2925 2926 if (io->async) { 2927 bool blocking = io->blocking; 2928 2929 fuse_aio_complete(io, ret < 0 ? ret : 0, -1); 2930 2931 /* we have a non-extending, async request, so return */ 2932 if (!blocking) 2933 return -EIOCBQUEUED; 2934 2935 wait_for_completion(&wait); 2936 ret = fuse_get_res_by_io(io); 2937 } 2938 2939 kref_put(&io->refcnt, fuse_io_release); 2940 2941 if (iov_iter_rw(iter) == WRITE) { 2942 if (ret > 0) 2943 fuse_write_update_size(inode, pos); 2944 else if (ret < 0 && offset + count > i_size) 2945 fuse_do_truncate(file); 2946 } 2947 2948 return ret; 2949 } 2950 2951 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, 2952 loff_t length) 2953 { 2954 struct fuse_file *ff = file->private_data; 2955 struct inode *inode = file_inode(file); 2956 struct fuse_inode *fi = get_fuse_inode(inode); 2957 struct fuse_conn *fc = ff->fc; 2958 FUSE_ARGS(args); 2959 struct fuse_fallocate_in inarg = { 2960 .fh = ff->fh, 2961 .offset = offset, 2962 .length = length, 2963 .mode = mode 2964 }; 2965 int err; 2966 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) || 2967 (mode & FALLOC_FL_PUNCH_HOLE); 2968 2969 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2970 return -EOPNOTSUPP; 2971 2972 if (fc->no_fallocate) 2973 return -EOPNOTSUPP; 2974 2975 if (lock_inode) { 2976 inode_lock(inode); 2977 if (mode & FALLOC_FL_PUNCH_HOLE) { 2978 loff_t endbyte = offset + length - 1; 2979 err = filemap_write_and_wait_range(inode->i_mapping, 2980 offset, endbyte); 2981 if (err) 2982 goto out; 2983 2984 fuse_sync_writes(inode); 2985 } 2986 } 2987 2988 if (!(mode & FALLOC_FL_KEEP_SIZE)) 2989 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 2990 2991 args.in.h.opcode = FUSE_FALLOCATE; 2992 args.in.h.nodeid = ff->nodeid; 2993 args.in.numargs = 1; 2994 args.in.args[0].size = sizeof(inarg); 2995 args.in.args[0].value = &inarg; 2996 err = fuse_simple_request(fc, &args); 2997 if (err == -ENOSYS) { 2998 fc->no_fallocate = 1; 2999 err = -EOPNOTSUPP; 3000 } 3001 if (err) 3002 goto out; 3003 3004 /* we could have extended the file */ 3005 if (!(mode & FALLOC_FL_KEEP_SIZE)) { 3006 bool changed = fuse_write_update_size(inode, offset + length); 3007 3008 if (changed && fc->writeback_cache) 3009 file_update_time(file); 3010 } 3011 3012 if (mode & FALLOC_FL_PUNCH_HOLE) 3013 truncate_pagecache_range(inode, offset, offset + length - 1); 3014 3015 fuse_invalidate_attr(inode); 3016 3017 out: 3018 if (!(mode & FALLOC_FL_KEEP_SIZE)) 3019 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 3020 3021 if (lock_inode) 3022 inode_unlock(inode); 3023 3024 return err; 3025 } 3026 3027 static ssize_t fuse_copy_file_range(struct file *file_in, loff_t pos_in, 3028 struct file *file_out, loff_t pos_out, 3029 size_t len, unsigned int flags) 3030 { 3031 struct fuse_file *ff_in = file_in->private_data; 3032 struct fuse_file *ff_out = file_out->private_data; 3033 struct inode *inode_out = file_inode(file_out); 3034 struct fuse_inode *fi_out = get_fuse_inode(inode_out); 3035 struct fuse_conn *fc = ff_in->fc; 3036 FUSE_ARGS(args); 3037 struct fuse_copy_file_range_in inarg = { 3038 .fh_in = ff_in->fh, 3039 .off_in = pos_in, 3040 .nodeid_out = ff_out->nodeid, 3041 .fh_out = ff_out->fh, 3042 .off_out = pos_out, 3043 .len = len, 3044 .flags = flags 3045 }; 3046 struct fuse_write_out outarg; 3047 ssize_t err; 3048 /* mark unstable when write-back is not used, and file_out gets 3049 * extended */ 3050 bool is_unstable = (!fc->writeback_cache) && 3051 ((pos_out + len) > inode_out->i_size); 3052 3053 if (fc->no_copy_file_range) 3054 return -EOPNOTSUPP; 3055 3056 inode_lock(inode_out); 3057 3058 if (fc->writeback_cache) { 3059 err = filemap_write_and_wait_range(inode_out->i_mapping, 3060 pos_out, pos_out + len); 3061 if (err) 3062 goto out; 3063 3064 fuse_sync_writes(inode_out); 3065 } 3066 3067 if (is_unstable) 3068 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); 3069 3070 args.in.h.opcode = FUSE_COPY_FILE_RANGE; 3071 args.in.h.nodeid = ff_in->nodeid; 3072 args.in.numargs = 1; 3073 args.in.args[0].size = sizeof(inarg); 3074 args.in.args[0].value = &inarg; 3075 args.out.numargs = 1; 3076 args.out.args[0].size = sizeof(outarg); 3077 args.out.args[0].value = &outarg; 3078 err = fuse_simple_request(fc, &args); 3079 if (err == -ENOSYS) { 3080 fc->no_copy_file_range = 1; 3081 err = -EOPNOTSUPP; 3082 } 3083 if (err) 3084 goto out; 3085 3086 if (fc->writeback_cache) { 3087 fuse_write_update_size(inode_out, pos_out + outarg.size); 3088 file_update_time(file_out); 3089 } 3090 3091 fuse_invalidate_attr(inode_out); 3092 3093 err = outarg.size; 3094 out: 3095 if (is_unstable) 3096 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); 3097 3098 inode_unlock(inode_out); 3099 3100 return err; 3101 } 3102 3103 static const struct file_operations fuse_file_operations = { 3104 .llseek = fuse_file_llseek, 3105 .read_iter = fuse_file_read_iter, 3106 .write_iter = fuse_file_write_iter, 3107 .mmap = fuse_file_mmap, 3108 .open = fuse_open, 3109 .flush = fuse_flush, 3110 .release = fuse_release, 3111 .fsync = fuse_fsync, 3112 .lock = fuse_file_lock, 3113 .flock = fuse_file_flock, 3114 .splice_read = generic_file_splice_read, 3115 .unlocked_ioctl = fuse_file_ioctl, 3116 .compat_ioctl = fuse_file_compat_ioctl, 3117 .poll = fuse_file_poll, 3118 .fallocate = fuse_file_fallocate, 3119 .copy_file_range = fuse_copy_file_range, 3120 }; 3121 3122 static const struct file_operations fuse_direct_io_file_operations = { 3123 .llseek = fuse_file_llseek, 3124 .read_iter = fuse_direct_read_iter, 3125 .write_iter = fuse_direct_write_iter, 3126 .mmap = fuse_direct_mmap, 3127 .open = fuse_open, 3128 .flush = fuse_flush, 3129 .release = fuse_release, 3130 .fsync = fuse_fsync, 3131 .lock = fuse_file_lock, 3132 .flock = fuse_file_flock, 3133 .unlocked_ioctl = fuse_file_ioctl, 3134 .compat_ioctl = fuse_file_compat_ioctl, 3135 .poll = fuse_file_poll, 3136 .fallocate = fuse_file_fallocate, 3137 /* no splice_read */ 3138 }; 3139 3140 static const struct address_space_operations fuse_file_aops = { 3141 .readpage = fuse_readpage, 3142 .writepage = fuse_writepage, 3143 .writepages = fuse_writepages, 3144 .launder_page = fuse_launder_page, 3145 .readpages = fuse_readpages, 3146 .set_page_dirty = __set_page_dirty_nobuffers, 3147 .bmap = fuse_bmap, 3148 .direct_IO = fuse_direct_IO, 3149 .write_begin = fuse_write_begin, 3150 .write_end = fuse_write_end, 3151 }; 3152 3153 void fuse_init_file_inode(struct inode *inode) 3154 { 3155 struct fuse_inode *fi = get_fuse_inode(inode); 3156 3157 inode->i_fop = &fuse_file_operations; 3158 inode->i_data.a_ops = &fuse_file_aops; 3159 3160 INIT_LIST_HEAD(&fi->write_files); 3161 INIT_LIST_HEAD(&fi->queued_writes); 3162 fi->writectr = 0; 3163 init_waitqueue_head(&fi->page_waitq); 3164 INIT_LIST_HEAD(&fi->writepages); 3165 } 3166