1 /* 2 FUSE: Filesystem in Userspace 3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> 4 5 This program can be distributed under the terms of the GNU GPL. 6 See the file COPYING. 7 */ 8 9 #include "fuse_i.h" 10 11 #include <linux/pagemap.h> 12 #include <linux/slab.h> 13 #include <linux/kernel.h> 14 #include <linux/sched.h> 15 #include <linux/sched/signal.h> 16 #include <linux/module.h> 17 #include <linux/compat.h> 18 #include <linux/swap.h> 19 #include <linux/falloc.h> 20 #include <linux/uio.h> 21 22 static const struct file_operations fuse_direct_io_file_operations; 23 24 static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file, 25 int opcode, struct fuse_open_out *outargp) 26 { 27 struct fuse_open_in inarg; 28 FUSE_ARGS(args); 29 30 memset(&inarg, 0, sizeof(inarg)); 31 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); 32 if (!fc->atomic_o_trunc) 33 inarg.flags &= ~O_TRUNC; 34 args.in.h.opcode = opcode; 35 args.in.h.nodeid = nodeid; 36 args.in.numargs = 1; 37 args.in.args[0].size = sizeof(inarg); 38 args.in.args[0].value = &inarg; 39 args.out.numargs = 1; 40 args.out.args[0].size = sizeof(*outargp); 41 args.out.args[0].value = outargp; 42 43 return fuse_simple_request(fc, &args); 44 } 45 46 struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) 47 { 48 struct fuse_file *ff; 49 50 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL); 51 if (unlikely(!ff)) 52 return NULL; 53 54 ff->fc = fc; 55 ff->reserved_req = fuse_request_alloc(0); 56 if (unlikely(!ff->reserved_req)) { 57 kfree(ff); 58 return NULL; 59 } 60 61 INIT_LIST_HEAD(&ff->write_entry); 62 refcount_set(&ff->count, 1); 63 RB_CLEAR_NODE(&ff->polled_node); 64 init_waitqueue_head(&ff->poll_wait); 65 66 spin_lock(&fc->lock); 67 ff->kh = ++fc->khctr; 68 spin_unlock(&fc->lock); 69 70 return ff; 71 } 72 73 void fuse_file_free(struct fuse_file *ff) 74 { 75 fuse_request_free(ff->reserved_req); 76 kfree(ff); 77 } 78 79 static struct fuse_file *fuse_file_get(struct fuse_file *ff) 80 { 81 refcount_inc(&ff->count); 82 return ff; 83 } 84 85 static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req) 86 { 87 iput(req->misc.release.inode); 88 } 89 90 static void fuse_file_put(struct fuse_file *ff, bool sync) 91 { 92 if (refcount_dec_and_test(&ff->count)) { 93 struct fuse_req *req = ff->reserved_req; 94 95 if (ff->fc->no_open) { 96 /* 97 * Drop the release request when client does not 98 * implement 'open' 99 */ 100 __clear_bit(FR_BACKGROUND, &req->flags); 101 iput(req->misc.release.inode); 102 fuse_put_request(ff->fc, req); 103 } else if (sync) { 104 __set_bit(FR_FORCE, &req->flags); 105 __clear_bit(FR_BACKGROUND, &req->flags); 106 fuse_request_send(ff->fc, req); 107 iput(req->misc.release.inode); 108 fuse_put_request(ff->fc, req); 109 } else { 110 req->end = fuse_release_end; 111 __set_bit(FR_BACKGROUND, &req->flags); 112 fuse_request_send_background(ff->fc, req); 113 } 114 kfree(ff); 115 } 116 } 117 118 int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file, 119 bool isdir) 120 { 121 struct fuse_file *ff; 122 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; 123 124 ff = fuse_file_alloc(fc); 125 if (!ff) 126 return -ENOMEM; 127 128 ff->fh = 0; 129 ff->open_flags = FOPEN_KEEP_CACHE; /* Default for no-open */ 130 if (!fc->no_open || isdir) { 131 struct fuse_open_out outarg; 132 int err; 133 134 err = fuse_send_open(fc, nodeid, file, opcode, &outarg); 135 if (!err) { 136 ff->fh = outarg.fh; 137 ff->open_flags = outarg.open_flags; 138 139 } else if (err != -ENOSYS || isdir) { 140 fuse_file_free(ff); 141 return err; 142 } else { 143 fc->no_open = 1; 144 } 145 } 146 147 if (isdir) 148 ff->open_flags &= ~FOPEN_DIRECT_IO; 149 150 ff->nodeid = nodeid; 151 file->private_data = ff; 152 153 return 0; 154 } 155 EXPORT_SYMBOL_GPL(fuse_do_open); 156 157 static void fuse_link_write_file(struct file *file) 158 { 159 struct inode *inode = file_inode(file); 160 struct fuse_conn *fc = get_fuse_conn(inode); 161 struct fuse_inode *fi = get_fuse_inode(inode); 162 struct fuse_file *ff = file->private_data; 163 /* 164 * file may be written through mmap, so chain it onto the 165 * inodes's write_file list 166 */ 167 spin_lock(&fc->lock); 168 if (list_empty(&ff->write_entry)) 169 list_add(&ff->write_entry, &fi->write_files); 170 spin_unlock(&fc->lock); 171 } 172 173 void fuse_finish_open(struct inode *inode, struct file *file) 174 { 175 struct fuse_file *ff = file->private_data; 176 struct fuse_conn *fc = get_fuse_conn(inode); 177 178 if (ff->open_flags & FOPEN_DIRECT_IO) 179 file->f_op = &fuse_direct_io_file_operations; 180 if (!(ff->open_flags & FOPEN_KEEP_CACHE)) 181 invalidate_inode_pages2(inode->i_mapping); 182 if (ff->open_flags & FOPEN_NONSEEKABLE) 183 nonseekable_open(inode, file); 184 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) { 185 struct fuse_inode *fi = get_fuse_inode(inode); 186 187 spin_lock(&fc->lock); 188 fi->attr_version = ++fc->attr_version; 189 i_size_write(inode, 0); 190 spin_unlock(&fc->lock); 191 fuse_invalidate_attr(inode); 192 if (fc->writeback_cache) 193 file_update_time(file); 194 } 195 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) 196 fuse_link_write_file(file); 197 } 198 199 int fuse_open_common(struct inode *inode, struct file *file, bool isdir) 200 { 201 struct fuse_conn *fc = get_fuse_conn(inode); 202 int err; 203 bool lock_inode = (file->f_flags & O_TRUNC) && 204 fc->atomic_o_trunc && 205 fc->writeback_cache; 206 207 err = generic_file_open(inode, file); 208 if (err) 209 return err; 210 211 if (lock_inode) 212 inode_lock(inode); 213 214 err = fuse_do_open(fc, get_node_id(inode), file, isdir); 215 216 if (!err) 217 fuse_finish_open(inode, file); 218 219 if (lock_inode) 220 inode_unlock(inode); 221 222 return err; 223 } 224 225 static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode) 226 { 227 struct fuse_conn *fc = ff->fc; 228 struct fuse_req *req = ff->reserved_req; 229 struct fuse_release_in *inarg = &req->misc.release.in; 230 231 spin_lock(&fc->lock); 232 list_del(&ff->write_entry); 233 if (!RB_EMPTY_NODE(&ff->polled_node)) 234 rb_erase(&ff->polled_node, &fc->polled_files); 235 spin_unlock(&fc->lock); 236 237 wake_up_interruptible_all(&ff->poll_wait); 238 239 inarg->fh = ff->fh; 240 inarg->flags = flags; 241 req->in.h.opcode = opcode; 242 req->in.h.nodeid = ff->nodeid; 243 req->in.numargs = 1; 244 req->in.args[0].size = sizeof(struct fuse_release_in); 245 req->in.args[0].value = inarg; 246 } 247 248 void fuse_release_common(struct file *file, int opcode) 249 { 250 struct fuse_file *ff = file->private_data; 251 struct fuse_req *req = ff->reserved_req; 252 253 fuse_prepare_release(ff, file->f_flags, opcode); 254 255 if (ff->flock) { 256 struct fuse_release_in *inarg = &req->misc.release.in; 257 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; 258 inarg->lock_owner = fuse_lock_owner_id(ff->fc, 259 (fl_owner_t) file); 260 } 261 /* Hold inode until release is finished */ 262 req->misc.release.inode = igrab(file_inode(file)); 263 264 /* 265 * Normally this will send the RELEASE request, however if 266 * some asynchronous READ or WRITE requests are outstanding, 267 * the sending will be delayed. 268 * 269 * Make the release synchronous if this is a fuseblk mount, 270 * synchronous RELEASE is allowed (and desirable) in this case 271 * because the server can be trusted not to screw up. 272 */ 273 fuse_file_put(ff, ff->fc->destroy_req != NULL); 274 } 275 276 static int fuse_open(struct inode *inode, struct file *file) 277 { 278 return fuse_open_common(inode, file, false); 279 } 280 281 static int fuse_release(struct inode *inode, struct file *file) 282 { 283 struct fuse_conn *fc = get_fuse_conn(inode); 284 285 /* see fuse_vma_close() for !writeback_cache case */ 286 if (fc->writeback_cache) 287 write_inode_now(inode, 1); 288 289 fuse_release_common(file, FUSE_RELEASE); 290 291 /* return value is ignored by VFS */ 292 return 0; 293 } 294 295 void fuse_sync_release(struct fuse_file *ff, int flags) 296 { 297 WARN_ON(refcount_read(&ff->count) > 1); 298 fuse_prepare_release(ff, flags, FUSE_RELEASE); 299 /* 300 * iput(NULL) is a no-op and since the refcount is 1 and everything's 301 * synchronous, we are fine with not doing igrab() here" 302 */ 303 fuse_file_put(ff, true); 304 } 305 EXPORT_SYMBOL_GPL(fuse_sync_release); 306 307 /* 308 * Scramble the ID space with XTEA, so that the value of the files_struct 309 * pointer is not exposed to userspace. 310 */ 311 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) 312 { 313 u32 *k = fc->scramble_key; 314 u64 v = (unsigned long) id; 315 u32 v0 = v; 316 u32 v1 = v >> 32; 317 u32 sum = 0; 318 int i; 319 320 for (i = 0; i < 32; i++) { 321 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]); 322 sum += 0x9E3779B9; 323 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]); 324 } 325 326 return (u64) v0 + ((u64) v1 << 32); 327 } 328 329 /* 330 * Check if any page in a range is under writeback 331 * 332 * This is currently done by walking the list of writepage requests 333 * for the inode, which can be pretty inefficient. 334 */ 335 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from, 336 pgoff_t idx_to) 337 { 338 struct fuse_conn *fc = get_fuse_conn(inode); 339 struct fuse_inode *fi = get_fuse_inode(inode); 340 struct fuse_req *req; 341 bool found = false; 342 343 spin_lock(&fc->lock); 344 list_for_each_entry(req, &fi->writepages, writepages_entry) { 345 pgoff_t curr_index; 346 347 BUG_ON(req->inode != inode); 348 curr_index = req->misc.write.in.offset >> PAGE_SHIFT; 349 if (idx_from < curr_index + req->num_pages && 350 curr_index <= idx_to) { 351 found = true; 352 break; 353 } 354 } 355 spin_unlock(&fc->lock); 356 357 return found; 358 } 359 360 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index) 361 { 362 return fuse_range_is_writeback(inode, index, index); 363 } 364 365 /* 366 * Wait for page writeback to be completed. 367 * 368 * Since fuse doesn't rely on the VM writeback tracking, this has to 369 * use some other means. 370 */ 371 static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) 372 { 373 struct fuse_inode *fi = get_fuse_inode(inode); 374 375 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); 376 return 0; 377 } 378 379 /* 380 * Wait for all pending writepages on the inode to finish. 381 * 382 * This is currently done by blocking further writes with FUSE_NOWRITE 383 * and waiting for all sent writes to complete. 384 * 385 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage 386 * could conflict with truncation. 387 */ 388 static void fuse_sync_writes(struct inode *inode) 389 { 390 fuse_set_nowrite(inode); 391 fuse_release_nowrite(inode); 392 } 393 394 static int fuse_flush(struct file *file, fl_owner_t id) 395 { 396 struct inode *inode = file_inode(file); 397 struct fuse_conn *fc = get_fuse_conn(inode); 398 struct fuse_file *ff = file->private_data; 399 struct fuse_req *req; 400 struct fuse_flush_in inarg; 401 int err; 402 403 if (is_bad_inode(inode)) 404 return -EIO; 405 406 if (fc->no_flush) 407 return 0; 408 409 err = write_inode_now(inode, 1); 410 if (err) 411 return err; 412 413 inode_lock(inode); 414 fuse_sync_writes(inode); 415 inode_unlock(inode); 416 417 err = filemap_check_errors(file->f_mapping); 418 if (err) 419 return err; 420 421 req = fuse_get_req_nofail_nopages(fc, file); 422 memset(&inarg, 0, sizeof(inarg)); 423 inarg.fh = ff->fh; 424 inarg.lock_owner = fuse_lock_owner_id(fc, id); 425 req->in.h.opcode = FUSE_FLUSH; 426 req->in.h.nodeid = get_node_id(inode); 427 req->in.numargs = 1; 428 req->in.args[0].size = sizeof(inarg); 429 req->in.args[0].value = &inarg; 430 __set_bit(FR_FORCE, &req->flags); 431 fuse_request_send(fc, req); 432 err = req->out.h.error; 433 fuse_put_request(fc, req); 434 if (err == -ENOSYS) { 435 fc->no_flush = 1; 436 err = 0; 437 } 438 return err; 439 } 440 441 int fuse_fsync_common(struct file *file, loff_t start, loff_t end, 442 int datasync, int isdir) 443 { 444 struct inode *inode = file->f_mapping->host; 445 struct fuse_conn *fc = get_fuse_conn(inode); 446 struct fuse_file *ff = file->private_data; 447 FUSE_ARGS(args); 448 struct fuse_fsync_in inarg; 449 int err; 450 451 if (is_bad_inode(inode)) 452 return -EIO; 453 454 inode_lock(inode); 455 456 /* 457 * Start writeback against all dirty pages of the inode, then 458 * wait for all outstanding writes, before sending the FSYNC 459 * request. 460 */ 461 err = file_write_and_wait_range(file, start, end); 462 if (err) 463 goto out; 464 465 fuse_sync_writes(inode); 466 467 /* 468 * Due to implementation of fuse writeback 469 * file_write_and_wait_range() does not catch errors. 470 * We have to do this directly after fuse_sync_writes() 471 */ 472 err = file_check_and_advance_wb_err(file); 473 if (err) 474 goto out; 475 476 err = sync_inode_metadata(inode, 1); 477 if (err) 478 goto out; 479 480 if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir)) 481 goto out; 482 483 memset(&inarg, 0, sizeof(inarg)); 484 inarg.fh = ff->fh; 485 inarg.fsync_flags = datasync ? 1 : 0; 486 args.in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC; 487 args.in.h.nodeid = get_node_id(inode); 488 args.in.numargs = 1; 489 args.in.args[0].size = sizeof(inarg); 490 args.in.args[0].value = &inarg; 491 err = fuse_simple_request(fc, &args); 492 if (err == -ENOSYS) { 493 if (isdir) 494 fc->no_fsyncdir = 1; 495 else 496 fc->no_fsync = 1; 497 err = 0; 498 } 499 out: 500 inode_unlock(inode); 501 return err; 502 } 503 504 static int fuse_fsync(struct file *file, loff_t start, loff_t end, 505 int datasync) 506 { 507 return fuse_fsync_common(file, start, end, datasync, 0); 508 } 509 510 void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos, 511 size_t count, int opcode) 512 { 513 struct fuse_read_in *inarg = &req->misc.read.in; 514 struct fuse_file *ff = file->private_data; 515 516 inarg->fh = ff->fh; 517 inarg->offset = pos; 518 inarg->size = count; 519 inarg->flags = file->f_flags; 520 req->in.h.opcode = opcode; 521 req->in.h.nodeid = ff->nodeid; 522 req->in.numargs = 1; 523 req->in.args[0].size = sizeof(struct fuse_read_in); 524 req->in.args[0].value = inarg; 525 req->out.argvar = 1; 526 req->out.numargs = 1; 527 req->out.args[0].size = count; 528 } 529 530 static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty) 531 { 532 unsigned i; 533 534 for (i = 0; i < req->num_pages; i++) { 535 struct page *page = req->pages[i]; 536 if (should_dirty) 537 set_page_dirty_lock(page); 538 put_page(page); 539 } 540 } 541 542 static void fuse_io_release(struct kref *kref) 543 { 544 kfree(container_of(kref, struct fuse_io_priv, refcnt)); 545 } 546 547 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) 548 { 549 if (io->err) 550 return io->err; 551 552 if (io->bytes >= 0 && io->write) 553 return -EIO; 554 555 return io->bytes < 0 ? io->size : io->bytes; 556 } 557 558 /** 559 * In case of short read, the caller sets 'pos' to the position of 560 * actual end of fuse request in IO request. Otherwise, if bytes_requested 561 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1. 562 * 563 * An example: 564 * User requested DIO read of 64K. It was splitted into two 32K fuse requests, 565 * both submitted asynchronously. The first of them was ACKed by userspace as 566 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The 567 * second request was ACKed as short, e.g. only 1K was read, resulting in 568 * pos == 33K. 569 * 570 * Thus, when all fuse requests are completed, the minimal non-negative 'pos' 571 * will be equal to the length of the longest contiguous fragment of 572 * transferred data starting from the beginning of IO request. 573 */ 574 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) 575 { 576 int left; 577 578 spin_lock(&io->lock); 579 if (err) 580 io->err = io->err ? : err; 581 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes)) 582 io->bytes = pos; 583 584 left = --io->reqs; 585 if (!left && io->blocking) 586 complete(io->done); 587 spin_unlock(&io->lock); 588 589 if (!left && !io->blocking) { 590 ssize_t res = fuse_get_res_by_io(io); 591 592 if (res >= 0) { 593 struct inode *inode = file_inode(io->iocb->ki_filp); 594 struct fuse_conn *fc = get_fuse_conn(inode); 595 struct fuse_inode *fi = get_fuse_inode(inode); 596 597 spin_lock(&fc->lock); 598 fi->attr_version = ++fc->attr_version; 599 spin_unlock(&fc->lock); 600 } 601 602 io->iocb->ki_complete(io->iocb, res, 0); 603 } 604 605 kref_put(&io->refcnt, fuse_io_release); 606 } 607 608 static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req) 609 { 610 struct fuse_io_priv *io = req->io; 611 ssize_t pos = -1; 612 613 fuse_release_user_pages(req, io->should_dirty); 614 615 if (io->write) { 616 if (req->misc.write.in.size != req->misc.write.out.size) 617 pos = req->misc.write.in.offset - io->offset + 618 req->misc.write.out.size; 619 } else { 620 if (req->misc.read.in.size != req->out.args[0].size) 621 pos = req->misc.read.in.offset - io->offset + 622 req->out.args[0].size; 623 } 624 625 fuse_aio_complete(io, req->out.h.error, pos); 626 } 627 628 static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req, 629 size_t num_bytes, struct fuse_io_priv *io) 630 { 631 spin_lock(&io->lock); 632 kref_get(&io->refcnt); 633 io->size += num_bytes; 634 io->reqs++; 635 spin_unlock(&io->lock); 636 637 req->io = io; 638 req->end = fuse_aio_complete_req; 639 640 __fuse_get_request(req); 641 fuse_request_send_background(fc, req); 642 643 return num_bytes; 644 } 645 646 static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io, 647 loff_t pos, size_t count, fl_owner_t owner) 648 { 649 struct file *file = io->iocb->ki_filp; 650 struct fuse_file *ff = file->private_data; 651 struct fuse_conn *fc = ff->fc; 652 653 fuse_read_fill(req, file, pos, count, FUSE_READ); 654 if (owner != NULL) { 655 struct fuse_read_in *inarg = &req->misc.read.in; 656 657 inarg->read_flags |= FUSE_READ_LOCKOWNER; 658 inarg->lock_owner = fuse_lock_owner_id(fc, owner); 659 } 660 661 if (io->async) 662 return fuse_async_req_send(fc, req, count, io); 663 664 fuse_request_send(fc, req); 665 return req->out.args[0].size; 666 } 667 668 static void fuse_read_update_size(struct inode *inode, loff_t size, 669 u64 attr_ver) 670 { 671 struct fuse_conn *fc = get_fuse_conn(inode); 672 struct fuse_inode *fi = get_fuse_inode(inode); 673 674 spin_lock(&fc->lock); 675 if (attr_ver == fi->attr_version && size < inode->i_size && 676 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { 677 fi->attr_version = ++fc->attr_version; 678 i_size_write(inode, size); 679 } 680 spin_unlock(&fc->lock); 681 } 682 683 static void fuse_short_read(struct fuse_req *req, struct inode *inode, 684 u64 attr_ver) 685 { 686 size_t num_read = req->out.args[0].size; 687 struct fuse_conn *fc = get_fuse_conn(inode); 688 689 if (fc->writeback_cache) { 690 /* 691 * A hole in a file. Some data after the hole are in page cache, 692 * but have not reached the client fs yet. So, the hole is not 693 * present there. 694 */ 695 int i; 696 int start_idx = num_read >> PAGE_SHIFT; 697 size_t off = num_read & (PAGE_SIZE - 1); 698 699 for (i = start_idx; i < req->num_pages; i++) { 700 zero_user_segment(req->pages[i], off, PAGE_SIZE); 701 off = 0; 702 } 703 } else { 704 loff_t pos = page_offset(req->pages[0]) + num_read; 705 fuse_read_update_size(inode, pos, attr_ver); 706 } 707 } 708 709 static int fuse_do_readpage(struct file *file, struct page *page) 710 { 711 struct kiocb iocb; 712 struct fuse_io_priv io; 713 struct inode *inode = page->mapping->host; 714 struct fuse_conn *fc = get_fuse_conn(inode); 715 struct fuse_req *req; 716 size_t num_read; 717 loff_t pos = page_offset(page); 718 size_t count = PAGE_SIZE; 719 u64 attr_ver; 720 int err; 721 722 /* 723 * Page writeback can extend beyond the lifetime of the 724 * page-cache page, so make sure we read a properly synced 725 * page. 726 */ 727 fuse_wait_on_page_writeback(inode, page->index); 728 729 req = fuse_get_req(fc, 1); 730 if (IS_ERR(req)) 731 return PTR_ERR(req); 732 733 attr_ver = fuse_get_attr_version(fc); 734 735 req->out.page_zeroing = 1; 736 req->out.argpages = 1; 737 req->num_pages = 1; 738 req->pages[0] = page; 739 req->page_descs[0].length = count; 740 init_sync_kiocb(&iocb, file); 741 io = (struct fuse_io_priv) FUSE_IO_PRIV_SYNC(&iocb); 742 num_read = fuse_send_read(req, &io, pos, count, NULL); 743 err = req->out.h.error; 744 745 if (!err) { 746 /* 747 * Short read means EOF. If file size is larger, truncate it 748 */ 749 if (num_read < count) 750 fuse_short_read(req, inode, attr_ver); 751 752 SetPageUptodate(page); 753 } 754 755 fuse_put_request(fc, req); 756 757 return err; 758 } 759 760 static int fuse_readpage(struct file *file, struct page *page) 761 { 762 struct inode *inode = page->mapping->host; 763 int err; 764 765 err = -EIO; 766 if (is_bad_inode(inode)) 767 goto out; 768 769 err = fuse_do_readpage(file, page); 770 fuse_invalidate_atime(inode); 771 out: 772 unlock_page(page); 773 return err; 774 } 775 776 static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req) 777 { 778 int i; 779 size_t count = req->misc.read.in.size; 780 size_t num_read = req->out.args[0].size; 781 struct address_space *mapping = NULL; 782 783 for (i = 0; mapping == NULL && i < req->num_pages; i++) 784 mapping = req->pages[i]->mapping; 785 786 if (mapping) { 787 struct inode *inode = mapping->host; 788 789 /* 790 * Short read means EOF. If file size is larger, truncate it 791 */ 792 if (!req->out.h.error && num_read < count) 793 fuse_short_read(req, inode, req->misc.read.attr_ver); 794 795 fuse_invalidate_atime(inode); 796 } 797 798 for (i = 0; i < req->num_pages; i++) { 799 struct page *page = req->pages[i]; 800 if (!req->out.h.error) 801 SetPageUptodate(page); 802 else 803 SetPageError(page); 804 unlock_page(page); 805 put_page(page); 806 } 807 if (req->ff) 808 fuse_file_put(req->ff, false); 809 } 810 811 static void fuse_send_readpages(struct fuse_req *req, struct file *file) 812 { 813 struct fuse_file *ff = file->private_data; 814 struct fuse_conn *fc = ff->fc; 815 loff_t pos = page_offset(req->pages[0]); 816 size_t count = req->num_pages << PAGE_SHIFT; 817 818 req->out.argpages = 1; 819 req->out.page_zeroing = 1; 820 req->out.page_replace = 1; 821 fuse_read_fill(req, file, pos, count, FUSE_READ); 822 req->misc.read.attr_ver = fuse_get_attr_version(fc); 823 if (fc->async_read) { 824 req->ff = fuse_file_get(ff); 825 req->end = fuse_readpages_end; 826 fuse_request_send_background(fc, req); 827 } else { 828 fuse_request_send(fc, req); 829 fuse_readpages_end(fc, req); 830 fuse_put_request(fc, req); 831 } 832 } 833 834 struct fuse_fill_data { 835 struct fuse_req *req; 836 struct file *file; 837 struct inode *inode; 838 unsigned nr_pages; 839 }; 840 841 static int fuse_readpages_fill(void *_data, struct page *page) 842 { 843 struct fuse_fill_data *data = _data; 844 struct fuse_req *req = data->req; 845 struct inode *inode = data->inode; 846 struct fuse_conn *fc = get_fuse_conn(inode); 847 848 fuse_wait_on_page_writeback(inode, page->index); 849 850 if (req->num_pages && 851 (req->num_pages == FUSE_MAX_PAGES_PER_REQ || 852 (req->num_pages + 1) * PAGE_SIZE > fc->max_read || 853 req->pages[req->num_pages - 1]->index + 1 != page->index)) { 854 int nr_alloc = min_t(unsigned, data->nr_pages, 855 FUSE_MAX_PAGES_PER_REQ); 856 fuse_send_readpages(req, data->file); 857 if (fc->async_read) 858 req = fuse_get_req_for_background(fc, nr_alloc); 859 else 860 req = fuse_get_req(fc, nr_alloc); 861 862 data->req = req; 863 if (IS_ERR(req)) { 864 unlock_page(page); 865 return PTR_ERR(req); 866 } 867 } 868 869 if (WARN_ON(req->num_pages >= req->max_pages)) { 870 fuse_put_request(fc, req); 871 return -EIO; 872 } 873 874 get_page(page); 875 req->pages[req->num_pages] = page; 876 req->page_descs[req->num_pages].length = PAGE_SIZE; 877 req->num_pages++; 878 data->nr_pages--; 879 return 0; 880 } 881 882 static int fuse_readpages(struct file *file, struct address_space *mapping, 883 struct list_head *pages, unsigned nr_pages) 884 { 885 struct inode *inode = mapping->host; 886 struct fuse_conn *fc = get_fuse_conn(inode); 887 struct fuse_fill_data data; 888 int err; 889 int nr_alloc = min_t(unsigned, nr_pages, FUSE_MAX_PAGES_PER_REQ); 890 891 err = -EIO; 892 if (is_bad_inode(inode)) 893 goto out; 894 895 data.file = file; 896 data.inode = inode; 897 if (fc->async_read) 898 data.req = fuse_get_req_for_background(fc, nr_alloc); 899 else 900 data.req = fuse_get_req(fc, nr_alloc); 901 data.nr_pages = nr_pages; 902 err = PTR_ERR(data.req); 903 if (IS_ERR(data.req)) 904 goto out; 905 906 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data); 907 if (!err) { 908 if (data.req->num_pages) 909 fuse_send_readpages(data.req, file); 910 else 911 fuse_put_request(fc, data.req); 912 } 913 out: 914 return err; 915 } 916 917 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 918 { 919 struct inode *inode = iocb->ki_filp->f_mapping->host; 920 struct fuse_conn *fc = get_fuse_conn(inode); 921 922 /* 923 * In auto invalidate mode, always update attributes on read. 924 * Otherwise, only update if we attempt to read past EOF (to ensure 925 * i_size is up to date). 926 */ 927 if (fc->auto_inval_data || 928 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { 929 int err; 930 err = fuse_update_attributes(inode, iocb->ki_filp); 931 if (err) 932 return err; 933 } 934 935 return generic_file_read_iter(iocb, to); 936 } 937 938 static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff, 939 loff_t pos, size_t count) 940 { 941 struct fuse_write_in *inarg = &req->misc.write.in; 942 struct fuse_write_out *outarg = &req->misc.write.out; 943 944 inarg->fh = ff->fh; 945 inarg->offset = pos; 946 inarg->size = count; 947 req->in.h.opcode = FUSE_WRITE; 948 req->in.h.nodeid = ff->nodeid; 949 req->in.numargs = 2; 950 if (ff->fc->minor < 9) 951 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; 952 else 953 req->in.args[0].size = sizeof(struct fuse_write_in); 954 req->in.args[0].value = inarg; 955 req->in.args[1].size = count; 956 req->out.numargs = 1; 957 req->out.args[0].size = sizeof(struct fuse_write_out); 958 req->out.args[0].value = outarg; 959 } 960 961 static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io, 962 loff_t pos, size_t count, fl_owner_t owner) 963 { 964 struct kiocb *iocb = io->iocb; 965 struct file *file = iocb->ki_filp; 966 struct fuse_file *ff = file->private_data; 967 struct fuse_conn *fc = ff->fc; 968 struct fuse_write_in *inarg = &req->misc.write.in; 969 970 fuse_write_fill(req, ff, pos, count); 971 inarg->flags = file->f_flags; 972 if (iocb->ki_flags & IOCB_DSYNC) 973 inarg->flags |= O_DSYNC; 974 if (iocb->ki_flags & IOCB_SYNC) 975 inarg->flags |= O_SYNC; 976 if (owner != NULL) { 977 inarg->write_flags |= FUSE_WRITE_LOCKOWNER; 978 inarg->lock_owner = fuse_lock_owner_id(fc, owner); 979 } 980 981 if (io->async) 982 return fuse_async_req_send(fc, req, count, io); 983 984 fuse_request_send(fc, req); 985 return req->misc.write.out.size; 986 } 987 988 bool fuse_write_update_size(struct inode *inode, loff_t pos) 989 { 990 struct fuse_conn *fc = get_fuse_conn(inode); 991 struct fuse_inode *fi = get_fuse_inode(inode); 992 bool ret = false; 993 994 spin_lock(&fc->lock); 995 fi->attr_version = ++fc->attr_version; 996 if (pos > inode->i_size) { 997 i_size_write(inode, pos); 998 ret = true; 999 } 1000 spin_unlock(&fc->lock); 1001 1002 return ret; 1003 } 1004 1005 static size_t fuse_send_write_pages(struct fuse_req *req, struct kiocb *iocb, 1006 struct inode *inode, loff_t pos, 1007 size_t count) 1008 { 1009 size_t res; 1010 unsigned offset; 1011 unsigned i; 1012 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1013 1014 for (i = 0; i < req->num_pages; i++) 1015 fuse_wait_on_page_writeback(inode, req->pages[i]->index); 1016 1017 res = fuse_send_write(req, &io, pos, count, NULL); 1018 1019 offset = req->page_descs[0].offset; 1020 count = res; 1021 for (i = 0; i < req->num_pages; i++) { 1022 struct page *page = req->pages[i]; 1023 1024 if (!req->out.h.error && !offset && count >= PAGE_SIZE) 1025 SetPageUptodate(page); 1026 1027 if (count > PAGE_SIZE - offset) 1028 count -= PAGE_SIZE - offset; 1029 else 1030 count = 0; 1031 offset = 0; 1032 1033 unlock_page(page); 1034 put_page(page); 1035 } 1036 1037 return res; 1038 } 1039 1040 static ssize_t fuse_fill_write_pages(struct fuse_req *req, 1041 struct address_space *mapping, 1042 struct iov_iter *ii, loff_t pos) 1043 { 1044 struct fuse_conn *fc = get_fuse_conn(mapping->host); 1045 unsigned offset = pos & (PAGE_SIZE - 1); 1046 size_t count = 0; 1047 int err; 1048 1049 req->in.argpages = 1; 1050 req->page_descs[0].offset = offset; 1051 1052 do { 1053 size_t tmp; 1054 struct page *page; 1055 pgoff_t index = pos >> PAGE_SHIFT; 1056 size_t bytes = min_t(size_t, PAGE_SIZE - offset, 1057 iov_iter_count(ii)); 1058 1059 bytes = min_t(size_t, bytes, fc->max_write - count); 1060 1061 again: 1062 err = -EFAULT; 1063 if (iov_iter_fault_in_readable(ii, bytes)) 1064 break; 1065 1066 err = -ENOMEM; 1067 page = grab_cache_page_write_begin(mapping, index, 0); 1068 if (!page) 1069 break; 1070 1071 if (mapping_writably_mapped(mapping)) 1072 flush_dcache_page(page); 1073 1074 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes); 1075 flush_dcache_page(page); 1076 1077 iov_iter_advance(ii, tmp); 1078 if (!tmp) { 1079 unlock_page(page); 1080 put_page(page); 1081 bytes = min(bytes, iov_iter_single_seg_count(ii)); 1082 goto again; 1083 } 1084 1085 err = 0; 1086 req->pages[req->num_pages] = page; 1087 req->page_descs[req->num_pages].length = tmp; 1088 req->num_pages++; 1089 1090 count += tmp; 1091 pos += tmp; 1092 offset += tmp; 1093 if (offset == PAGE_SIZE) 1094 offset = 0; 1095 1096 if (!fc->big_writes) 1097 break; 1098 } while (iov_iter_count(ii) && count < fc->max_write && 1099 req->num_pages < req->max_pages && offset == 0); 1100 1101 return count > 0 ? count : err; 1102 } 1103 1104 static inline unsigned fuse_wr_pages(loff_t pos, size_t len) 1105 { 1106 return min_t(unsigned, 1107 ((pos + len - 1) >> PAGE_SHIFT) - 1108 (pos >> PAGE_SHIFT) + 1, 1109 FUSE_MAX_PAGES_PER_REQ); 1110 } 1111 1112 static ssize_t fuse_perform_write(struct kiocb *iocb, 1113 struct address_space *mapping, 1114 struct iov_iter *ii, loff_t pos) 1115 { 1116 struct inode *inode = mapping->host; 1117 struct fuse_conn *fc = get_fuse_conn(inode); 1118 struct fuse_inode *fi = get_fuse_inode(inode); 1119 int err = 0; 1120 ssize_t res = 0; 1121 1122 if (is_bad_inode(inode)) 1123 return -EIO; 1124 1125 if (inode->i_size < pos + iov_iter_count(ii)) 1126 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1127 1128 do { 1129 struct fuse_req *req; 1130 ssize_t count; 1131 unsigned nr_pages = fuse_wr_pages(pos, iov_iter_count(ii)); 1132 1133 req = fuse_get_req(fc, nr_pages); 1134 if (IS_ERR(req)) { 1135 err = PTR_ERR(req); 1136 break; 1137 } 1138 1139 count = fuse_fill_write_pages(req, mapping, ii, pos); 1140 if (count <= 0) { 1141 err = count; 1142 } else { 1143 size_t num_written; 1144 1145 num_written = fuse_send_write_pages(req, iocb, inode, 1146 pos, count); 1147 err = req->out.h.error; 1148 if (!err) { 1149 res += num_written; 1150 pos += num_written; 1151 1152 /* break out of the loop on short write */ 1153 if (num_written != count) 1154 err = -EIO; 1155 } 1156 } 1157 fuse_put_request(fc, req); 1158 } while (!err && iov_iter_count(ii)); 1159 1160 if (res > 0) 1161 fuse_write_update_size(inode, pos); 1162 1163 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1164 fuse_invalidate_attr(inode); 1165 1166 return res > 0 ? res : err; 1167 } 1168 1169 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 1170 { 1171 struct file *file = iocb->ki_filp; 1172 struct address_space *mapping = file->f_mapping; 1173 ssize_t written = 0; 1174 ssize_t written_buffered = 0; 1175 struct inode *inode = mapping->host; 1176 ssize_t err; 1177 loff_t endbyte = 0; 1178 1179 if (get_fuse_conn(inode)->writeback_cache) { 1180 /* Update size (EOF optimization) and mode (SUID clearing) */ 1181 err = fuse_update_attributes(mapping->host, file); 1182 if (err) 1183 return err; 1184 1185 return generic_file_write_iter(iocb, from); 1186 } 1187 1188 inode_lock(inode); 1189 1190 /* We can write back this queue in page reclaim */ 1191 current->backing_dev_info = inode_to_bdi(inode); 1192 1193 err = generic_write_checks(iocb, from); 1194 if (err <= 0) 1195 goto out; 1196 1197 err = file_remove_privs(file); 1198 if (err) 1199 goto out; 1200 1201 err = file_update_time(file); 1202 if (err) 1203 goto out; 1204 1205 if (iocb->ki_flags & IOCB_DIRECT) { 1206 loff_t pos = iocb->ki_pos; 1207 written = generic_file_direct_write(iocb, from); 1208 if (written < 0 || !iov_iter_count(from)) 1209 goto out; 1210 1211 pos += written; 1212 1213 written_buffered = fuse_perform_write(iocb, mapping, from, pos); 1214 if (written_buffered < 0) { 1215 err = written_buffered; 1216 goto out; 1217 } 1218 endbyte = pos + written_buffered - 1; 1219 1220 err = filemap_write_and_wait_range(file->f_mapping, pos, 1221 endbyte); 1222 if (err) 1223 goto out; 1224 1225 invalidate_mapping_pages(file->f_mapping, 1226 pos >> PAGE_SHIFT, 1227 endbyte >> PAGE_SHIFT); 1228 1229 written += written_buffered; 1230 iocb->ki_pos = pos + written_buffered; 1231 } else { 1232 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos); 1233 if (written >= 0) 1234 iocb->ki_pos += written; 1235 } 1236 out: 1237 current->backing_dev_info = NULL; 1238 inode_unlock(inode); 1239 if (written > 0) 1240 written = generic_write_sync(iocb, written); 1241 1242 return written ? written : err; 1243 } 1244 1245 static inline void fuse_page_descs_length_init(struct fuse_req *req, 1246 unsigned index, unsigned nr_pages) 1247 { 1248 int i; 1249 1250 for (i = index; i < index + nr_pages; i++) 1251 req->page_descs[i].length = PAGE_SIZE - 1252 req->page_descs[i].offset; 1253 } 1254 1255 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) 1256 { 1257 return (unsigned long)ii->iov->iov_base + ii->iov_offset; 1258 } 1259 1260 static inline size_t fuse_get_frag_size(const struct iov_iter *ii, 1261 size_t max_size) 1262 { 1263 return min(iov_iter_single_seg_count(ii), max_size); 1264 } 1265 1266 static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii, 1267 size_t *nbytesp, int write) 1268 { 1269 size_t nbytes = 0; /* # bytes already packed in req */ 1270 ssize_t ret = 0; 1271 1272 /* Special case for kernel I/O: can copy directly into the buffer */ 1273 if (ii->type & ITER_KVEC) { 1274 unsigned long user_addr = fuse_get_user_addr(ii); 1275 size_t frag_size = fuse_get_frag_size(ii, *nbytesp); 1276 1277 if (write) 1278 req->in.args[1].value = (void *) user_addr; 1279 else 1280 req->out.args[0].value = (void *) user_addr; 1281 1282 iov_iter_advance(ii, frag_size); 1283 *nbytesp = frag_size; 1284 return 0; 1285 } 1286 1287 while (nbytes < *nbytesp && req->num_pages < req->max_pages) { 1288 unsigned npages; 1289 size_t start; 1290 ret = iov_iter_get_pages(ii, &req->pages[req->num_pages], 1291 *nbytesp - nbytes, 1292 req->max_pages - req->num_pages, 1293 &start); 1294 if (ret < 0) 1295 break; 1296 1297 iov_iter_advance(ii, ret); 1298 nbytes += ret; 1299 1300 ret += start; 1301 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE; 1302 1303 req->page_descs[req->num_pages].offset = start; 1304 fuse_page_descs_length_init(req, req->num_pages, npages); 1305 1306 req->num_pages += npages; 1307 req->page_descs[req->num_pages - 1].length -= 1308 (PAGE_SIZE - ret) & (PAGE_SIZE - 1); 1309 } 1310 1311 if (write) 1312 req->in.argpages = 1; 1313 else 1314 req->out.argpages = 1; 1315 1316 *nbytesp = nbytes; 1317 1318 return ret < 0 ? ret : 0; 1319 } 1320 1321 static inline int fuse_iter_npages(const struct iov_iter *ii_p) 1322 { 1323 return iov_iter_npages(ii_p, FUSE_MAX_PAGES_PER_REQ); 1324 } 1325 1326 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, 1327 loff_t *ppos, int flags) 1328 { 1329 int write = flags & FUSE_DIO_WRITE; 1330 int cuse = flags & FUSE_DIO_CUSE; 1331 struct file *file = io->iocb->ki_filp; 1332 struct inode *inode = file->f_mapping->host; 1333 struct fuse_file *ff = file->private_data; 1334 struct fuse_conn *fc = ff->fc; 1335 size_t nmax = write ? fc->max_write : fc->max_read; 1336 loff_t pos = *ppos; 1337 size_t count = iov_iter_count(iter); 1338 pgoff_t idx_from = pos >> PAGE_SHIFT; 1339 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT; 1340 ssize_t res = 0; 1341 struct fuse_req *req; 1342 int err = 0; 1343 1344 if (io->async) 1345 req = fuse_get_req_for_background(fc, fuse_iter_npages(iter)); 1346 else 1347 req = fuse_get_req(fc, fuse_iter_npages(iter)); 1348 if (IS_ERR(req)) 1349 return PTR_ERR(req); 1350 1351 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) { 1352 if (!write) 1353 inode_lock(inode); 1354 fuse_sync_writes(inode); 1355 if (!write) 1356 inode_unlock(inode); 1357 } 1358 1359 io->should_dirty = !write && iter_is_iovec(iter); 1360 while (count) { 1361 size_t nres; 1362 fl_owner_t owner = current->files; 1363 size_t nbytes = min(count, nmax); 1364 err = fuse_get_user_pages(req, iter, &nbytes, write); 1365 if (err && !nbytes) 1366 break; 1367 1368 if (write) 1369 nres = fuse_send_write(req, io, pos, nbytes, owner); 1370 else 1371 nres = fuse_send_read(req, io, pos, nbytes, owner); 1372 1373 if (!io->async) 1374 fuse_release_user_pages(req, io->should_dirty); 1375 if (req->out.h.error) { 1376 err = req->out.h.error; 1377 break; 1378 } else if (nres > nbytes) { 1379 res = 0; 1380 err = -EIO; 1381 break; 1382 } 1383 count -= nres; 1384 res += nres; 1385 pos += nres; 1386 if (nres != nbytes) 1387 break; 1388 if (count) { 1389 fuse_put_request(fc, req); 1390 if (io->async) 1391 req = fuse_get_req_for_background(fc, 1392 fuse_iter_npages(iter)); 1393 else 1394 req = fuse_get_req(fc, fuse_iter_npages(iter)); 1395 if (IS_ERR(req)) 1396 break; 1397 } 1398 } 1399 if (!IS_ERR(req)) 1400 fuse_put_request(fc, req); 1401 if (res > 0) 1402 *ppos = pos; 1403 1404 return res > 0 ? res : err; 1405 } 1406 EXPORT_SYMBOL_GPL(fuse_direct_io); 1407 1408 static ssize_t __fuse_direct_read(struct fuse_io_priv *io, 1409 struct iov_iter *iter, 1410 loff_t *ppos) 1411 { 1412 ssize_t res; 1413 struct inode *inode = file_inode(io->iocb->ki_filp); 1414 1415 if (is_bad_inode(inode)) 1416 return -EIO; 1417 1418 res = fuse_direct_io(io, iter, ppos, 0); 1419 1420 fuse_invalidate_attr(inode); 1421 1422 return res; 1423 } 1424 1425 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) 1426 { 1427 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1428 return __fuse_direct_read(&io, to, &iocb->ki_pos); 1429 } 1430 1431 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) 1432 { 1433 struct inode *inode = file_inode(iocb->ki_filp); 1434 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1435 ssize_t res; 1436 1437 if (is_bad_inode(inode)) 1438 return -EIO; 1439 1440 /* Don't allow parallel writes to the same file */ 1441 inode_lock(inode); 1442 res = generic_write_checks(iocb, from); 1443 if (res > 0) 1444 res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); 1445 fuse_invalidate_attr(inode); 1446 if (res > 0) 1447 fuse_write_update_size(inode, iocb->ki_pos); 1448 inode_unlock(inode); 1449 1450 return res; 1451 } 1452 1453 static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req) 1454 { 1455 int i; 1456 1457 for (i = 0; i < req->num_pages; i++) 1458 __free_page(req->pages[i]); 1459 1460 if (req->ff) 1461 fuse_file_put(req->ff, false); 1462 } 1463 1464 static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req) 1465 { 1466 struct inode *inode = req->inode; 1467 struct fuse_inode *fi = get_fuse_inode(inode); 1468 struct backing_dev_info *bdi = inode_to_bdi(inode); 1469 int i; 1470 1471 list_del(&req->writepages_entry); 1472 for (i = 0; i < req->num_pages; i++) { 1473 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 1474 dec_node_page_state(req->pages[i], NR_WRITEBACK_TEMP); 1475 wb_writeout_inc(&bdi->wb); 1476 } 1477 wake_up(&fi->page_waitq); 1478 } 1479 1480 /* Called under fc->lock, may release and reacquire it */ 1481 static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req, 1482 loff_t size) 1483 __releases(fc->lock) 1484 __acquires(fc->lock) 1485 { 1486 struct fuse_inode *fi = get_fuse_inode(req->inode); 1487 struct fuse_write_in *inarg = &req->misc.write.in; 1488 __u64 data_size = req->num_pages * PAGE_SIZE; 1489 1490 if (!fc->connected) 1491 goto out_free; 1492 1493 if (inarg->offset + data_size <= size) { 1494 inarg->size = data_size; 1495 } else if (inarg->offset < size) { 1496 inarg->size = size - inarg->offset; 1497 } else { 1498 /* Got truncated off completely */ 1499 goto out_free; 1500 } 1501 1502 req->in.args[1].size = inarg->size; 1503 fi->writectr++; 1504 fuse_request_send_background_locked(fc, req); 1505 return; 1506 1507 out_free: 1508 fuse_writepage_finish(fc, req); 1509 spin_unlock(&fc->lock); 1510 fuse_writepage_free(fc, req); 1511 fuse_put_request(fc, req); 1512 spin_lock(&fc->lock); 1513 } 1514 1515 /* 1516 * If fi->writectr is positive (no truncate or fsync going on) send 1517 * all queued writepage requests. 1518 * 1519 * Called with fc->lock 1520 */ 1521 void fuse_flush_writepages(struct inode *inode) 1522 __releases(fc->lock) 1523 __acquires(fc->lock) 1524 { 1525 struct fuse_conn *fc = get_fuse_conn(inode); 1526 struct fuse_inode *fi = get_fuse_inode(inode); 1527 size_t crop = i_size_read(inode); 1528 struct fuse_req *req; 1529 1530 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { 1531 req = list_entry(fi->queued_writes.next, struct fuse_req, list); 1532 list_del_init(&req->list); 1533 fuse_send_writepage(fc, req, crop); 1534 } 1535 } 1536 1537 static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req) 1538 { 1539 struct inode *inode = req->inode; 1540 struct fuse_inode *fi = get_fuse_inode(inode); 1541 1542 mapping_set_error(inode->i_mapping, req->out.h.error); 1543 spin_lock(&fc->lock); 1544 while (req->misc.write.next) { 1545 struct fuse_conn *fc = get_fuse_conn(inode); 1546 struct fuse_write_in *inarg = &req->misc.write.in; 1547 struct fuse_req *next = req->misc.write.next; 1548 req->misc.write.next = next->misc.write.next; 1549 next->misc.write.next = NULL; 1550 next->ff = fuse_file_get(req->ff); 1551 list_add(&next->writepages_entry, &fi->writepages); 1552 1553 /* 1554 * Skip fuse_flush_writepages() to make it easy to crop requests 1555 * based on primary request size. 1556 * 1557 * 1st case (trivial): there are no concurrent activities using 1558 * fuse_set/release_nowrite. Then we're on safe side because 1559 * fuse_flush_writepages() would call fuse_send_writepage() 1560 * anyway. 1561 * 1562 * 2nd case: someone called fuse_set_nowrite and it is waiting 1563 * now for completion of all in-flight requests. This happens 1564 * rarely and no more than once per page, so this should be 1565 * okay. 1566 * 1567 * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle 1568 * of fuse_set_nowrite..fuse_release_nowrite section. The fact 1569 * that fuse_set_nowrite returned implies that all in-flight 1570 * requests were completed along with all of their secondary 1571 * requests. Further primary requests are blocked by negative 1572 * writectr. Hence there cannot be any in-flight requests and 1573 * no invocations of fuse_writepage_end() while we're in 1574 * fuse_set_nowrite..fuse_release_nowrite section. 1575 */ 1576 fuse_send_writepage(fc, next, inarg->offset + inarg->size); 1577 } 1578 fi->writectr--; 1579 fuse_writepage_finish(fc, req); 1580 spin_unlock(&fc->lock); 1581 fuse_writepage_free(fc, req); 1582 } 1583 1584 static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc, 1585 struct fuse_inode *fi) 1586 { 1587 struct fuse_file *ff = NULL; 1588 1589 spin_lock(&fc->lock); 1590 if (!list_empty(&fi->write_files)) { 1591 ff = list_entry(fi->write_files.next, struct fuse_file, 1592 write_entry); 1593 fuse_file_get(ff); 1594 } 1595 spin_unlock(&fc->lock); 1596 1597 return ff; 1598 } 1599 1600 static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc, 1601 struct fuse_inode *fi) 1602 { 1603 struct fuse_file *ff = __fuse_write_file_get(fc, fi); 1604 WARN_ON(!ff); 1605 return ff; 1606 } 1607 1608 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) 1609 { 1610 struct fuse_conn *fc = get_fuse_conn(inode); 1611 struct fuse_inode *fi = get_fuse_inode(inode); 1612 struct fuse_file *ff; 1613 int err; 1614 1615 ff = __fuse_write_file_get(fc, fi); 1616 err = fuse_flush_times(inode, ff); 1617 if (ff) 1618 fuse_file_put(ff, 0); 1619 1620 return err; 1621 } 1622 1623 static int fuse_writepage_locked(struct page *page) 1624 { 1625 struct address_space *mapping = page->mapping; 1626 struct inode *inode = mapping->host; 1627 struct fuse_conn *fc = get_fuse_conn(inode); 1628 struct fuse_inode *fi = get_fuse_inode(inode); 1629 struct fuse_req *req; 1630 struct page *tmp_page; 1631 int error = -ENOMEM; 1632 1633 set_page_writeback(page); 1634 1635 req = fuse_request_alloc_nofs(1); 1636 if (!req) 1637 goto err; 1638 1639 /* writeback always goes to bg_queue */ 1640 __set_bit(FR_BACKGROUND, &req->flags); 1641 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 1642 if (!tmp_page) 1643 goto err_free; 1644 1645 error = -EIO; 1646 req->ff = fuse_write_file_get(fc, fi); 1647 if (!req->ff) 1648 goto err_nofile; 1649 1650 fuse_write_fill(req, req->ff, page_offset(page), 0); 1651 1652 copy_highpage(tmp_page, page); 1653 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE; 1654 req->misc.write.next = NULL; 1655 req->in.argpages = 1; 1656 req->num_pages = 1; 1657 req->pages[0] = tmp_page; 1658 req->page_descs[0].offset = 0; 1659 req->page_descs[0].length = PAGE_SIZE; 1660 req->end = fuse_writepage_end; 1661 req->inode = inode; 1662 1663 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 1664 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 1665 1666 spin_lock(&fc->lock); 1667 list_add(&req->writepages_entry, &fi->writepages); 1668 list_add_tail(&req->list, &fi->queued_writes); 1669 fuse_flush_writepages(inode); 1670 spin_unlock(&fc->lock); 1671 1672 end_page_writeback(page); 1673 1674 return 0; 1675 1676 err_nofile: 1677 __free_page(tmp_page); 1678 err_free: 1679 fuse_request_free(req); 1680 err: 1681 mapping_set_error(page->mapping, error); 1682 end_page_writeback(page); 1683 return error; 1684 } 1685 1686 static int fuse_writepage(struct page *page, struct writeback_control *wbc) 1687 { 1688 int err; 1689 1690 if (fuse_page_is_writeback(page->mapping->host, page->index)) { 1691 /* 1692 * ->writepages() should be called for sync() and friends. We 1693 * should only get here on direct reclaim and then we are 1694 * allowed to skip a page which is already in flight 1695 */ 1696 WARN_ON(wbc->sync_mode == WB_SYNC_ALL); 1697 1698 redirty_page_for_writepage(wbc, page); 1699 return 0; 1700 } 1701 1702 err = fuse_writepage_locked(page); 1703 unlock_page(page); 1704 1705 return err; 1706 } 1707 1708 struct fuse_fill_wb_data { 1709 struct fuse_req *req; 1710 struct fuse_file *ff; 1711 struct inode *inode; 1712 struct page **orig_pages; 1713 }; 1714 1715 static void fuse_writepages_send(struct fuse_fill_wb_data *data) 1716 { 1717 struct fuse_req *req = data->req; 1718 struct inode *inode = data->inode; 1719 struct fuse_conn *fc = get_fuse_conn(inode); 1720 struct fuse_inode *fi = get_fuse_inode(inode); 1721 int num_pages = req->num_pages; 1722 int i; 1723 1724 req->ff = fuse_file_get(data->ff); 1725 spin_lock(&fc->lock); 1726 list_add_tail(&req->list, &fi->queued_writes); 1727 fuse_flush_writepages(inode); 1728 spin_unlock(&fc->lock); 1729 1730 for (i = 0; i < num_pages; i++) 1731 end_page_writeback(data->orig_pages[i]); 1732 } 1733 1734 static bool fuse_writepage_in_flight(struct fuse_req *new_req, 1735 struct page *page) 1736 { 1737 struct fuse_conn *fc = get_fuse_conn(new_req->inode); 1738 struct fuse_inode *fi = get_fuse_inode(new_req->inode); 1739 struct fuse_req *tmp; 1740 struct fuse_req *old_req; 1741 bool found = false; 1742 pgoff_t curr_index; 1743 1744 BUG_ON(new_req->num_pages != 0); 1745 1746 spin_lock(&fc->lock); 1747 list_del(&new_req->writepages_entry); 1748 list_for_each_entry(old_req, &fi->writepages, writepages_entry) { 1749 BUG_ON(old_req->inode != new_req->inode); 1750 curr_index = old_req->misc.write.in.offset >> PAGE_SHIFT; 1751 if (curr_index <= page->index && 1752 page->index < curr_index + old_req->num_pages) { 1753 found = true; 1754 break; 1755 } 1756 } 1757 if (!found) { 1758 list_add(&new_req->writepages_entry, &fi->writepages); 1759 goto out_unlock; 1760 } 1761 1762 new_req->num_pages = 1; 1763 for (tmp = old_req; tmp != NULL; tmp = tmp->misc.write.next) { 1764 BUG_ON(tmp->inode != new_req->inode); 1765 curr_index = tmp->misc.write.in.offset >> PAGE_SHIFT; 1766 if (tmp->num_pages == 1 && 1767 curr_index == page->index) { 1768 old_req = tmp; 1769 } 1770 } 1771 1772 if (old_req->num_pages == 1 && test_bit(FR_PENDING, &old_req->flags)) { 1773 struct backing_dev_info *bdi = inode_to_bdi(page->mapping->host); 1774 1775 copy_highpage(old_req->pages[0], page); 1776 spin_unlock(&fc->lock); 1777 1778 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 1779 dec_node_page_state(page, NR_WRITEBACK_TEMP); 1780 wb_writeout_inc(&bdi->wb); 1781 fuse_writepage_free(fc, new_req); 1782 fuse_request_free(new_req); 1783 goto out; 1784 } else { 1785 new_req->misc.write.next = old_req->misc.write.next; 1786 old_req->misc.write.next = new_req; 1787 } 1788 out_unlock: 1789 spin_unlock(&fc->lock); 1790 out: 1791 return found; 1792 } 1793 1794 static int fuse_writepages_fill(struct page *page, 1795 struct writeback_control *wbc, void *_data) 1796 { 1797 struct fuse_fill_wb_data *data = _data; 1798 struct fuse_req *req = data->req; 1799 struct inode *inode = data->inode; 1800 struct fuse_conn *fc = get_fuse_conn(inode); 1801 struct page *tmp_page; 1802 bool is_writeback; 1803 int err; 1804 1805 if (!data->ff) { 1806 err = -EIO; 1807 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode)); 1808 if (!data->ff) 1809 goto out_unlock; 1810 } 1811 1812 /* 1813 * Being under writeback is unlikely but possible. For example direct 1814 * read to an mmaped fuse file will set the page dirty twice; once when 1815 * the pages are faulted with get_user_pages(), and then after the read 1816 * completed. 1817 */ 1818 is_writeback = fuse_page_is_writeback(inode, page->index); 1819 1820 if (req && req->num_pages && 1821 (is_writeback || req->num_pages == FUSE_MAX_PAGES_PER_REQ || 1822 (req->num_pages + 1) * PAGE_SIZE > fc->max_write || 1823 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) { 1824 fuse_writepages_send(data); 1825 data->req = NULL; 1826 } 1827 err = -ENOMEM; 1828 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 1829 if (!tmp_page) 1830 goto out_unlock; 1831 1832 /* 1833 * The page must not be redirtied until the writeout is completed 1834 * (i.e. userspace has sent a reply to the write request). Otherwise 1835 * there could be more than one temporary page instance for each real 1836 * page. 1837 * 1838 * This is ensured by holding the page lock in page_mkwrite() while 1839 * checking fuse_page_is_writeback(). We already hold the page lock 1840 * since clear_page_dirty_for_io() and keep it held until we add the 1841 * request to the fi->writepages list and increment req->num_pages. 1842 * After this fuse_page_is_writeback() will indicate that the page is 1843 * under writeback, so we can release the page lock. 1844 */ 1845 if (data->req == NULL) { 1846 struct fuse_inode *fi = get_fuse_inode(inode); 1847 1848 err = -ENOMEM; 1849 req = fuse_request_alloc_nofs(FUSE_MAX_PAGES_PER_REQ); 1850 if (!req) { 1851 __free_page(tmp_page); 1852 goto out_unlock; 1853 } 1854 1855 fuse_write_fill(req, data->ff, page_offset(page), 0); 1856 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE; 1857 req->misc.write.next = NULL; 1858 req->in.argpages = 1; 1859 __set_bit(FR_BACKGROUND, &req->flags); 1860 req->num_pages = 0; 1861 req->end = fuse_writepage_end; 1862 req->inode = inode; 1863 1864 spin_lock(&fc->lock); 1865 list_add(&req->writepages_entry, &fi->writepages); 1866 spin_unlock(&fc->lock); 1867 1868 data->req = req; 1869 } 1870 set_page_writeback(page); 1871 1872 copy_highpage(tmp_page, page); 1873 req->pages[req->num_pages] = tmp_page; 1874 req->page_descs[req->num_pages].offset = 0; 1875 req->page_descs[req->num_pages].length = PAGE_SIZE; 1876 1877 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 1878 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 1879 1880 err = 0; 1881 if (is_writeback && fuse_writepage_in_flight(req, page)) { 1882 end_page_writeback(page); 1883 data->req = NULL; 1884 goto out_unlock; 1885 } 1886 data->orig_pages[req->num_pages] = page; 1887 1888 /* 1889 * Protected by fc->lock against concurrent access by 1890 * fuse_page_is_writeback(). 1891 */ 1892 spin_lock(&fc->lock); 1893 req->num_pages++; 1894 spin_unlock(&fc->lock); 1895 1896 out_unlock: 1897 unlock_page(page); 1898 1899 return err; 1900 } 1901 1902 static int fuse_writepages(struct address_space *mapping, 1903 struct writeback_control *wbc) 1904 { 1905 struct inode *inode = mapping->host; 1906 struct fuse_fill_wb_data data; 1907 int err; 1908 1909 err = -EIO; 1910 if (is_bad_inode(inode)) 1911 goto out; 1912 1913 data.inode = inode; 1914 data.req = NULL; 1915 data.ff = NULL; 1916 1917 err = -ENOMEM; 1918 data.orig_pages = kcalloc(FUSE_MAX_PAGES_PER_REQ, 1919 sizeof(struct page *), 1920 GFP_NOFS); 1921 if (!data.orig_pages) 1922 goto out; 1923 1924 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); 1925 if (data.req) { 1926 /* Ignore errors if we can write at least one page */ 1927 BUG_ON(!data.req->num_pages); 1928 fuse_writepages_send(&data); 1929 err = 0; 1930 } 1931 if (data.ff) 1932 fuse_file_put(data.ff, false); 1933 1934 kfree(data.orig_pages); 1935 out: 1936 return err; 1937 } 1938 1939 /* 1940 * It's worthy to make sure that space is reserved on disk for the write, 1941 * but how to implement it without killing performance need more thinking. 1942 */ 1943 static int fuse_write_begin(struct file *file, struct address_space *mapping, 1944 loff_t pos, unsigned len, unsigned flags, 1945 struct page **pagep, void **fsdata) 1946 { 1947 pgoff_t index = pos >> PAGE_SHIFT; 1948 struct fuse_conn *fc = get_fuse_conn(file_inode(file)); 1949 struct page *page; 1950 loff_t fsize; 1951 int err = -ENOMEM; 1952 1953 WARN_ON(!fc->writeback_cache); 1954 1955 page = grab_cache_page_write_begin(mapping, index, flags); 1956 if (!page) 1957 goto error; 1958 1959 fuse_wait_on_page_writeback(mapping->host, page->index); 1960 1961 if (PageUptodate(page) || len == PAGE_SIZE) 1962 goto success; 1963 /* 1964 * Check if the start this page comes after the end of file, in which 1965 * case the readpage can be optimized away. 1966 */ 1967 fsize = i_size_read(mapping->host); 1968 if (fsize <= (pos & PAGE_MASK)) { 1969 size_t off = pos & ~PAGE_MASK; 1970 if (off) 1971 zero_user_segment(page, 0, off); 1972 goto success; 1973 } 1974 err = fuse_do_readpage(file, page); 1975 if (err) 1976 goto cleanup; 1977 success: 1978 *pagep = page; 1979 return 0; 1980 1981 cleanup: 1982 unlock_page(page); 1983 put_page(page); 1984 error: 1985 return err; 1986 } 1987 1988 static int fuse_write_end(struct file *file, struct address_space *mapping, 1989 loff_t pos, unsigned len, unsigned copied, 1990 struct page *page, void *fsdata) 1991 { 1992 struct inode *inode = page->mapping->host; 1993 1994 /* Haven't copied anything? Skip zeroing, size extending, dirtying. */ 1995 if (!copied) 1996 goto unlock; 1997 1998 if (!PageUptodate(page)) { 1999 /* Zero any unwritten bytes at the end of the page */ 2000 size_t endoff = (pos + copied) & ~PAGE_MASK; 2001 if (endoff) 2002 zero_user_segment(page, endoff, PAGE_SIZE); 2003 SetPageUptodate(page); 2004 } 2005 2006 fuse_write_update_size(inode, pos + copied); 2007 set_page_dirty(page); 2008 2009 unlock: 2010 unlock_page(page); 2011 put_page(page); 2012 2013 return copied; 2014 } 2015 2016 static int fuse_launder_page(struct page *page) 2017 { 2018 int err = 0; 2019 if (clear_page_dirty_for_io(page)) { 2020 struct inode *inode = page->mapping->host; 2021 err = fuse_writepage_locked(page); 2022 if (!err) 2023 fuse_wait_on_page_writeback(inode, page->index); 2024 } 2025 return err; 2026 } 2027 2028 /* 2029 * Write back dirty pages now, because there may not be any suitable 2030 * open files later 2031 */ 2032 static void fuse_vma_close(struct vm_area_struct *vma) 2033 { 2034 filemap_write_and_wait(vma->vm_file->f_mapping); 2035 } 2036 2037 /* 2038 * Wait for writeback against this page to complete before allowing it 2039 * to be marked dirty again, and hence written back again, possibly 2040 * before the previous writepage completed. 2041 * 2042 * Block here, instead of in ->writepage(), so that the userspace fs 2043 * can only block processes actually operating on the filesystem. 2044 * 2045 * Otherwise unprivileged userspace fs would be able to block 2046 * unrelated: 2047 * 2048 * - page migration 2049 * - sync(2) 2050 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER 2051 */ 2052 static int fuse_page_mkwrite(struct vm_fault *vmf) 2053 { 2054 struct page *page = vmf->page; 2055 struct inode *inode = file_inode(vmf->vma->vm_file); 2056 2057 file_update_time(vmf->vma->vm_file); 2058 lock_page(page); 2059 if (page->mapping != inode->i_mapping) { 2060 unlock_page(page); 2061 return VM_FAULT_NOPAGE; 2062 } 2063 2064 fuse_wait_on_page_writeback(inode, page->index); 2065 return VM_FAULT_LOCKED; 2066 } 2067 2068 static const struct vm_operations_struct fuse_file_vm_ops = { 2069 .close = fuse_vma_close, 2070 .fault = filemap_fault, 2071 .map_pages = filemap_map_pages, 2072 .page_mkwrite = fuse_page_mkwrite, 2073 }; 2074 2075 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) 2076 { 2077 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 2078 fuse_link_write_file(file); 2079 2080 file_accessed(file); 2081 vma->vm_ops = &fuse_file_vm_ops; 2082 return 0; 2083 } 2084 2085 static int fuse_direct_mmap(struct file *file, struct vm_area_struct *vma) 2086 { 2087 /* Can't provide the coherency needed for MAP_SHARED */ 2088 if (vma->vm_flags & VM_MAYSHARE) 2089 return -ENODEV; 2090 2091 invalidate_inode_pages2(file->f_mapping); 2092 2093 return generic_file_mmap(file, vma); 2094 } 2095 2096 static int convert_fuse_file_lock(struct fuse_conn *fc, 2097 const struct fuse_file_lock *ffl, 2098 struct file_lock *fl) 2099 { 2100 switch (ffl->type) { 2101 case F_UNLCK: 2102 break; 2103 2104 case F_RDLCK: 2105 case F_WRLCK: 2106 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX || 2107 ffl->end < ffl->start) 2108 return -EIO; 2109 2110 fl->fl_start = ffl->start; 2111 fl->fl_end = ffl->end; 2112 2113 /* 2114 * Convert pid into init's pid namespace. The locks API will 2115 * translate it into the caller's pid namespace. 2116 */ 2117 rcu_read_lock(); 2118 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns); 2119 rcu_read_unlock(); 2120 break; 2121 2122 default: 2123 return -EIO; 2124 } 2125 fl->fl_type = ffl->type; 2126 return 0; 2127 } 2128 2129 static void fuse_lk_fill(struct fuse_args *args, struct file *file, 2130 const struct file_lock *fl, int opcode, pid_t pid, 2131 int flock, struct fuse_lk_in *inarg) 2132 { 2133 struct inode *inode = file_inode(file); 2134 struct fuse_conn *fc = get_fuse_conn(inode); 2135 struct fuse_file *ff = file->private_data; 2136 2137 memset(inarg, 0, sizeof(*inarg)); 2138 inarg->fh = ff->fh; 2139 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner); 2140 inarg->lk.start = fl->fl_start; 2141 inarg->lk.end = fl->fl_end; 2142 inarg->lk.type = fl->fl_type; 2143 inarg->lk.pid = pid; 2144 if (flock) 2145 inarg->lk_flags |= FUSE_LK_FLOCK; 2146 args->in.h.opcode = opcode; 2147 args->in.h.nodeid = get_node_id(inode); 2148 args->in.numargs = 1; 2149 args->in.args[0].size = sizeof(*inarg); 2150 args->in.args[0].value = inarg; 2151 } 2152 2153 static int fuse_getlk(struct file *file, struct file_lock *fl) 2154 { 2155 struct inode *inode = file_inode(file); 2156 struct fuse_conn *fc = get_fuse_conn(inode); 2157 FUSE_ARGS(args); 2158 struct fuse_lk_in inarg; 2159 struct fuse_lk_out outarg; 2160 int err; 2161 2162 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg); 2163 args.out.numargs = 1; 2164 args.out.args[0].size = sizeof(outarg); 2165 args.out.args[0].value = &outarg; 2166 err = fuse_simple_request(fc, &args); 2167 if (!err) 2168 err = convert_fuse_file_lock(fc, &outarg.lk, fl); 2169 2170 return err; 2171 } 2172 2173 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) 2174 { 2175 struct inode *inode = file_inode(file); 2176 struct fuse_conn *fc = get_fuse_conn(inode); 2177 FUSE_ARGS(args); 2178 struct fuse_lk_in inarg; 2179 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; 2180 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL; 2181 pid_t pid_nr = pid_nr_ns(pid, fc->pid_ns); 2182 int err; 2183 2184 if (fl->fl_lmops && fl->fl_lmops->lm_grant) { 2185 /* NLM needs asynchronous locks, which we don't support yet */ 2186 return -ENOLCK; 2187 } 2188 2189 /* Unlock on close is handled by the flush method */ 2190 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX) 2191 return 0; 2192 2193 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); 2194 err = fuse_simple_request(fc, &args); 2195 2196 /* locking is restartable */ 2197 if (err == -EINTR) 2198 err = -ERESTARTSYS; 2199 2200 return err; 2201 } 2202 2203 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) 2204 { 2205 struct inode *inode = file_inode(file); 2206 struct fuse_conn *fc = get_fuse_conn(inode); 2207 int err; 2208 2209 if (cmd == F_CANCELLK) { 2210 err = 0; 2211 } else if (cmd == F_GETLK) { 2212 if (fc->no_lock) { 2213 posix_test_lock(file, fl); 2214 err = 0; 2215 } else 2216 err = fuse_getlk(file, fl); 2217 } else { 2218 if (fc->no_lock) 2219 err = posix_lock_file(file, fl, NULL); 2220 else 2221 err = fuse_setlk(file, fl, 0); 2222 } 2223 return err; 2224 } 2225 2226 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) 2227 { 2228 struct inode *inode = file_inode(file); 2229 struct fuse_conn *fc = get_fuse_conn(inode); 2230 int err; 2231 2232 if (fc->no_flock) { 2233 err = locks_lock_file_wait(file, fl); 2234 } else { 2235 struct fuse_file *ff = file->private_data; 2236 2237 /* emulate flock with POSIX locks */ 2238 ff->flock = true; 2239 err = fuse_setlk(file, fl, 1); 2240 } 2241 2242 return err; 2243 } 2244 2245 static sector_t fuse_bmap(struct address_space *mapping, sector_t block) 2246 { 2247 struct inode *inode = mapping->host; 2248 struct fuse_conn *fc = get_fuse_conn(inode); 2249 FUSE_ARGS(args); 2250 struct fuse_bmap_in inarg; 2251 struct fuse_bmap_out outarg; 2252 int err; 2253 2254 if (!inode->i_sb->s_bdev || fc->no_bmap) 2255 return 0; 2256 2257 memset(&inarg, 0, sizeof(inarg)); 2258 inarg.block = block; 2259 inarg.blocksize = inode->i_sb->s_blocksize; 2260 args.in.h.opcode = FUSE_BMAP; 2261 args.in.h.nodeid = get_node_id(inode); 2262 args.in.numargs = 1; 2263 args.in.args[0].size = sizeof(inarg); 2264 args.in.args[0].value = &inarg; 2265 args.out.numargs = 1; 2266 args.out.args[0].size = sizeof(outarg); 2267 args.out.args[0].value = &outarg; 2268 err = fuse_simple_request(fc, &args); 2269 if (err == -ENOSYS) 2270 fc->no_bmap = 1; 2271 2272 return err ? 0 : outarg.block; 2273 } 2274 2275 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) 2276 { 2277 struct inode *inode = file->f_mapping->host; 2278 struct fuse_conn *fc = get_fuse_conn(inode); 2279 struct fuse_file *ff = file->private_data; 2280 FUSE_ARGS(args); 2281 struct fuse_lseek_in inarg = { 2282 .fh = ff->fh, 2283 .offset = offset, 2284 .whence = whence 2285 }; 2286 struct fuse_lseek_out outarg; 2287 int err; 2288 2289 if (fc->no_lseek) 2290 goto fallback; 2291 2292 args.in.h.opcode = FUSE_LSEEK; 2293 args.in.h.nodeid = ff->nodeid; 2294 args.in.numargs = 1; 2295 args.in.args[0].size = sizeof(inarg); 2296 args.in.args[0].value = &inarg; 2297 args.out.numargs = 1; 2298 args.out.args[0].size = sizeof(outarg); 2299 args.out.args[0].value = &outarg; 2300 err = fuse_simple_request(fc, &args); 2301 if (err) { 2302 if (err == -ENOSYS) { 2303 fc->no_lseek = 1; 2304 goto fallback; 2305 } 2306 return err; 2307 } 2308 2309 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes); 2310 2311 fallback: 2312 err = fuse_update_attributes(inode, file); 2313 if (!err) 2314 return generic_file_llseek(file, offset, whence); 2315 else 2316 return err; 2317 } 2318 2319 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) 2320 { 2321 loff_t retval; 2322 struct inode *inode = file_inode(file); 2323 2324 switch (whence) { 2325 case SEEK_SET: 2326 case SEEK_CUR: 2327 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */ 2328 retval = generic_file_llseek(file, offset, whence); 2329 break; 2330 case SEEK_END: 2331 inode_lock(inode); 2332 retval = fuse_update_attributes(inode, file); 2333 if (!retval) 2334 retval = generic_file_llseek(file, offset, whence); 2335 inode_unlock(inode); 2336 break; 2337 case SEEK_HOLE: 2338 case SEEK_DATA: 2339 inode_lock(inode); 2340 retval = fuse_lseek(file, offset, whence); 2341 inode_unlock(inode); 2342 break; 2343 default: 2344 retval = -EINVAL; 2345 } 2346 2347 return retval; 2348 } 2349 2350 /* 2351 * CUSE servers compiled on 32bit broke on 64bit kernels because the 2352 * ABI was defined to be 'struct iovec' which is different on 32bit 2353 * and 64bit. Fortunately we can determine which structure the server 2354 * used from the size of the reply. 2355 */ 2356 static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src, 2357 size_t transferred, unsigned count, 2358 bool is_compat) 2359 { 2360 #ifdef CONFIG_COMPAT 2361 if (count * sizeof(struct compat_iovec) == transferred) { 2362 struct compat_iovec *ciov = src; 2363 unsigned i; 2364 2365 /* 2366 * With this interface a 32bit server cannot support 2367 * non-compat (i.e. ones coming from 64bit apps) ioctl 2368 * requests 2369 */ 2370 if (!is_compat) 2371 return -EINVAL; 2372 2373 for (i = 0; i < count; i++) { 2374 dst[i].iov_base = compat_ptr(ciov[i].iov_base); 2375 dst[i].iov_len = ciov[i].iov_len; 2376 } 2377 return 0; 2378 } 2379 #endif 2380 2381 if (count * sizeof(struct iovec) != transferred) 2382 return -EIO; 2383 2384 memcpy(dst, src, transferred); 2385 return 0; 2386 } 2387 2388 /* Make sure iov_length() won't overflow */ 2389 static int fuse_verify_ioctl_iov(struct iovec *iov, size_t count) 2390 { 2391 size_t n; 2392 u32 max = FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT; 2393 2394 for (n = 0; n < count; n++, iov++) { 2395 if (iov->iov_len > (size_t) max) 2396 return -ENOMEM; 2397 max -= iov->iov_len; 2398 } 2399 return 0; 2400 } 2401 2402 static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst, 2403 void *src, size_t transferred, unsigned count, 2404 bool is_compat) 2405 { 2406 unsigned i; 2407 struct fuse_ioctl_iovec *fiov = src; 2408 2409 if (fc->minor < 16) { 2410 return fuse_copy_ioctl_iovec_old(dst, src, transferred, 2411 count, is_compat); 2412 } 2413 2414 if (count * sizeof(struct fuse_ioctl_iovec) != transferred) 2415 return -EIO; 2416 2417 for (i = 0; i < count; i++) { 2418 /* Did the server supply an inappropriate value? */ 2419 if (fiov[i].base != (unsigned long) fiov[i].base || 2420 fiov[i].len != (unsigned long) fiov[i].len) 2421 return -EIO; 2422 2423 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base; 2424 dst[i].iov_len = (size_t) fiov[i].len; 2425 2426 #ifdef CONFIG_COMPAT 2427 if (is_compat && 2428 (ptr_to_compat(dst[i].iov_base) != fiov[i].base || 2429 (compat_size_t) dst[i].iov_len != fiov[i].len)) 2430 return -EIO; 2431 #endif 2432 } 2433 2434 return 0; 2435 } 2436 2437 2438 /* 2439 * For ioctls, there is no generic way to determine how much memory 2440 * needs to be read and/or written. Furthermore, ioctls are allowed 2441 * to dereference the passed pointer, so the parameter requires deep 2442 * copying but FUSE has no idea whatsoever about what to copy in or 2443 * out. 2444 * 2445 * This is solved by allowing FUSE server to retry ioctl with 2446 * necessary in/out iovecs. Let's assume the ioctl implementation 2447 * needs to read in the following structure. 2448 * 2449 * struct a { 2450 * char *buf; 2451 * size_t buflen; 2452 * } 2453 * 2454 * On the first callout to FUSE server, inarg->in_size and 2455 * inarg->out_size will be NULL; then, the server completes the ioctl 2456 * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and 2457 * the actual iov array to 2458 * 2459 * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) } } 2460 * 2461 * which tells FUSE to copy in the requested area and retry the ioctl. 2462 * On the second round, the server has access to the structure and 2463 * from that it can tell what to look for next, so on the invocation, 2464 * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to 2465 * 2466 * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) }, 2467 * { .iov_base = a.buf, .iov_len = a.buflen } } 2468 * 2469 * FUSE will copy both struct a and the pointed buffer from the 2470 * process doing the ioctl and retry ioctl with both struct a and the 2471 * buffer. 2472 * 2473 * This time, FUSE server has everything it needs and completes ioctl 2474 * without FUSE_IOCTL_RETRY which finishes the ioctl call. 2475 * 2476 * Copying data out works the same way. 2477 * 2478 * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel 2479 * automatically initializes in and out iovs by decoding @cmd with 2480 * _IOC_* macros and the server is not allowed to request RETRY. This 2481 * limits ioctl data transfers to well-formed ioctls and is the forced 2482 * behavior for all FUSE servers. 2483 */ 2484 long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, 2485 unsigned int flags) 2486 { 2487 struct fuse_file *ff = file->private_data; 2488 struct fuse_conn *fc = ff->fc; 2489 struct fuse_ioctl_in inarg = { 2490 .fh = ff->fh, 2491 .cmd = cmd, 2492 .arg = arg, 2493 .flags = flags 2494 }; 2495 struct fuse_ioctl_out outarg; 2496 struct fuse_req *req = NULL; 2497 struct page **pages = NULL; 2498 struct iovec *iov_page = NULL; 2499 struct iovec *in_iov = NULL, *out_iov = NULL; 2500 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages; 2501 size_t in_size, out_size, transferred, c; 2502 int err, i; 2503 struct iov_iter ii; 2504 2505 #if BITS_PER_LONG == 32 2506 inarg.flags |= FUSE_IOCTL_32BIT; 2507 #else 2508 if (flags & FUSE_IOCTL_COMPAT) 2509 inarg.flags |= FUSE_IOCTL_32BIT; 2510 #endif 2511 2512 /* assume all the iovs returned by client always fits in a page */ 2513 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); 2514 2515 err = -ENOMEM; 2516 pages = kcalloc(FUSE_MAX_PAGES_PER_REQ, sizeof(pages[0]), GFP_KERNEL); 2517 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); 2518 if (!pages || !iov_page) 2519 goto out; 2520 2521 /* 2522 * If restricted, initialize IO parameters as encoded in @cmd. 2523 * RETRY from server is not allowed. 2524 */ 2525 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) { 2526 struct iovec *iov = iov_page; 2527 2528 iov->iov_base = (void __user *)arg; 2529 iov->iov_len = _IOC_SIZE(cmd); 2530 2531 if (_IOC_DIR(cmd) & _IOC_WRITE) { 2532 in_iov = iov; 2533 in_iovs = 1; 2534 } 2535 2536 if (_IOC_DIR(cmd) & _IOC_READ) { 2537 out_iov = iov; 2538 out_iovs = 1; 2539 } 2540 } 2541 2542 retry: 2543 inarg.in_size = in_size = iov_length(in_iov, in_iovs); 2544 inarg.out_size = out_size = iov_length(out_iov, out_iovs); 2545 2546 /* 2547 * Out data can be used either for actual out data or iovs, 2548 * make sure there always is at least one page. 2549 */ 2550 out_size = max_t(size_t, out_size, PAGE_SIZE); 2551 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE); 2552 2553 /* make sure there are enough buffer pages and init request with them */ 2554 err = -ENOMEM; 2555 if (max_pages > FUSE_MAX_PAGES_PER_REQ) 2556 goto out; 2557 while (num_pages < max_pages) { 2558 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); 2559 if (!pages[num_pages]) 2560 goto out; 2561 num_pages++; 2562 } 2563 2564 req = fuse_get_req(fc, num_pages); 2565 if (IS_ERR(req)) { 2566 err = PTR_ERR(req); 2567 req = NULL; 2568 goto out; 2569 } 2570 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages); 2571 req->num_pages = num_pages; 2572 fuse_page_descs_length_init(req, 0, req->num_pages); 2573 2574 /* okay, let's send it to the client */ 2575 req->in.h.opcode = FUSE_IOCTL; 2576 req->in.h.nodeid = ff->nodeid; 2577 req->in.numargs = 1; 2578 req->in.args[0].size = sizeof(inarg); 2579 req->in.args[0].value = &inarg; 2580 if (in_size) { 2581 req->in.numargs++; 2582 req->in.args[1].size = in_size; 2583 req->in.argpages = 1; 2584 2585 err = -EFAULT; 2586 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size); 2587 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) { 2588 c = copy_page_from_iter(pages[i], 0, PAGE_SIZE, &ii); 2589 if (c != PAGE_SIZE && iov_iter_count(&ii)) 2590 goto out; 2591 } 2592 } 2593 2594 req->out.numargs = 2; 2595 req->out.args[0].size = sizeof(outarg); 2596 req->out.args[0].value = &outarg; 2597 req->out.args[1].size = out_size; 2598 req->out.argpages = 1; 2599 req->out.argvar = 1; 2600 2601 fuse_request_send(fc, req); 2602 err = req->out.h.error; 2603 transferred = req->out.args[1].size; 2604 fuse_put_request(fc, req); 2605 req = NULL; 2606 if (err) 2607 goto out; 2608 2609 /* did it ask for retry? */ 2610 if (outarg.flags & FUSE_IOCTL_RETRY) { 2611 void *vaddr; 2612 2613 /* no retry if in restricted mode */ 2614 err = -EIO; 2615 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) 2616 goto out; 2617 2618 in_iovs = outarg.in_iovs; 2619 out_iovs = outarg.out_iovs; 2620 2621 /* 2622 * Make sure things are in boundary, separate checks 2623 * are to protect against overflow. 2624 */ 2625 err = -ENOMEM; 2626 if (in_iovs > FUSE_IOCTL_MAX_IOV || 2627 out_iovs > FUSE_IOCTL_MAX_IOV || 2628 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) 2629 goto out; 2630 2631 vaddr = kmap_atomic(pages[0]); 2632 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr, 2633 transferred, in_iovs + out_iovs, 2634 (flags & FUSE_IOCTL_COMPAT) != 0); 2635 kunmap_atomic(vaddr); 2636 if (err) 2637 goto out; 2638 2639 in_iov = iov_page; 2640 out_iov = in_iov + in_iovs; 2641 2642 err = fuse_verify_ioctl_iov(in_iov, in_iovs); 2643 if (err) 2644 goto out; 2645 2646 err = fuse_verify_ioctl_iov(out_iov, out_iovs); 2647 if (err) 2648 goto out; 2649 2650 goto retry; 2651 } 2652 2653 err = -EIO; 2654 if (transferred > inarg.out_size) 2655 goto out; 2656 2657 err = -EFAULT; 2658 iov_iter_init(&ii, READ, out_iov, out_iovs, transferred); 2659 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) { 2660 c = copy_page_to_iter(pages[i], 0, PAGE_SIZE, &ii); 2661 if (c != PAGE_SIZE && iov_iter_count(&ii)) 2662 goto out; 2663 } 2664 err = 0; 2665 out: 2666 if (req) 2667 fuse_put_request(fc, req); 2668 free_page((unsigned long) iov_page); 2669 while (num_pages) 2670 __free_page(pages[--num_pages]); 2671 kfree(pages); 2672 2673 return err ? err : outarg.result; 2674 } 2675 EXPORT_SYMBOL_GPL(fuse_do_ioctl); 2676 2677 long fuse_ioctl_common(struct file *file, unsigned int cmd, 2678 unsigned long arg, unsigned int flags) 2679 { 2680 struct inode *inode = file_inode(file); 2681 struct fuse_conn *fc = get_fuse_conn(inode); 2682 2683 if (!fuse_allow_current_process(fc)) 2684 return -EACCES; 2685 2686 if (is_bad_inode(inode)) 2687 return -EIO; 2688 2689 return fuse_do_ioctl(file, cmd, arg, flags); 2690 } 2691 2692 static long fuse_file_ioctl(struct file *file, unsigned int cmd, 2693 unsigned long arg) 2694 { 2695 return fuse_ioctl_common(file, cmd, arg, 0); 2696 } 2697 2698 static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd, 2699 unsigned long arg) 2700 { 2701 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT); 2702 } 2703 2704 /* 2705 * All files which have been polled are linked to RB tree 2706 * fuse_conn->polled_files which is indexed by kh. Walk the tree and 2707 * find the matching one. 2708 */ 2709 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, 2710 struct rb_node **parent_out) 2711 { 2712 struct rb_node **link = &fc->polled_files.rb_node; 2713 struct rb_node *last = NULL; 2714 2715 while (*link) { 2716 struct fuse_file *ff; 2717 2718 last = *link; 2719 ff = rb_entry(last, struct fuse_file, polled_node); 2720 2721 if (kh < ff->kh) 2722 link = &last->rb_left; 2723 else if (kh > ff->kh) 2724 link = &last->rb_right; 2725 else 2726 return link; 2727 } 2728 2729 if (parent_out) 2730 *parent_out = last; 2731 return link; 2732 } 2733 2734 /* 2735 * The file is about to be polled. Make sure it's on the polled_files 2736 * RB tree. Note that files once added to the polled_files tree are 2737 * not removed before the file is released. This is because a file 2738 * polled once is likely to be polled again. 2739 */ 2740 static void fuse_register_polled_file(struct fuse_conn *fc, 2741 struct fuse_file *ff) 2742 { 2743 spin_lock(&fc->lock); 2744 if (RB_EMPTY_NODE(&ff->polled_node)) { 2745 struct rb_node **link, *uninitialized_var(parent); 2746 2747 link = fuse_find_polled_node(fc, ff->kh, &parent); 2748 BUG_ON(*link); 2749 rb_link_node(&ff->polled_node, parent, link); 2750 rb_insert_color(&ff->polled_node, &fc->polled_files); 2751 } 2752 spin_unlock(&fc->lock); 2753 } 2754 2755 __poll_t fuse_file_poll(struct file *file, poll_table *wait) 2756 { 2757 struct fuse_file *ff = file->private_data; 2758 struct fuse_conn *fc = ff->fc; 2759 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; 2760 struct fuse_poll_out outarg; 2761 FUSE_ARGS(args); 2762 int err; 2763 2764 if (fc->no_poll) 2765 return DEFAULT_POLLMASK; 2766 2767 poll_wait(file, &ff->poll_wait, wait); 2768 inarg.events = mangle_poll(poll_requested_events(wait)); 2769 2770 /* 2771 * Ask for notification iff there's someone waiting for it. 2772 * The client may ignore the flag and always notify. 2773 */ 2774 if (waitqueue_active(&ff->poll_wait)) { 2775 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; 2776 fuse_register_polled_file(fc, ff); 2777 } 2778 2779 args.in.h.opcode = FUSE_POLL; 2780 args.in.h.nodeid = ff->nodeid; 2781 args.in.numargs = 1; 2782 args.in.args[0].size = sizeof(inarg); 2783 args.in.args[0].value = &inarg; 2784 args.out.numargs = 1; 2785 args.out.args[0].size = sizeof(outarg); 2786 args.out.args[0].value = &outarg; 2787 err = fuse_simple_request(fc, &args); 2788 2789 if (!err) 2790 return demangle_poll(outarg.revents); 2791 if (err == -ENOSYS) { 2792 fc->no_poll = 1; 2793 return DEFAULT_POLLMASK; 2794 } 2795 return EPOLLERR; 2796 } 2797 EXPORT_SYMBOL_GPL(fuse_file_poll); 2798 2799 /* 2800 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and 2801 * wakes up the poll waiters. 2802 */ 2803 int fuse_notify_poll_wakeup(struct fuse_conn *fc, 2804 struct fuse_notify_poll_wakeup_out *outarg) 2805 { 2806 u64 kh = outarg->kh; 2807 struct rb_node **link; 2808 2809 spin_lock(&fc->lock); 2810 2811 link = fuse_find_polled_node(fc, kh, NULL); 2812 if (*link) { 2813 struct fuse_file *ff; 2814 2815 ff = rb_entry(*link, struct fuse_file, polled_node); 2816 wake_up_interruptible_sync(&ff->poll_wait); 2817 } 2818 2819 spin_unlock(&fc->lock); 2820 return 0; 2821 } 2822 2823 static void fuse_do_truncate(struct file *file) 2824 { 2825 struct inode *inode = file->f_mapping->host; 2826 struct iattr attr; 2827 2828 attr.ia_valid = ATTR_SIZE; 2829 attr.ia_size = i_size_read(inode); 2830 2831 attr.ia_file = file; 2832 attr.ia_valid |= ATTR_FILE; 2833 2834 fuse_do_setattr(file_dentry(file), &attr, file); 2835 } 2836 2837 static inline loff_t fuse_round_up(loff_t off) 2838 { 2839 return round_up(off, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT); 2840 } 2841 2842 static ssize_t 2843 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 2844 { 2845 DECLARE_COMPLETION_ONSTACK(wait); 2846 ssize_t ret = 0; 2847 struct file *file = iocb->ki_filp; 2848 struct fuse_file *ff = file->private_data; 2849 bool async_dio = ff->fc->async_dio; 2850 loff_t pos = 0; 2851 struct inode *inode; 2852 loff_t i_size; 2853 size_t count = iov_iter_count(iter); 2854 loff_t offset = iocb->ki_pos; 2855 struct fuse_io_priv *io; 2856 2857 pos = offset; 2858 inode = file->f_mapping->host; 2859 i_size = i_size_read(inode); 2860 2861 if ((iov_iter_rw(iter) == READ) && (offset > i_size)) 2862 return 0; 2863 2864 /* optimization for short read */ 2865 if (async_dio && iov_iter_rw(iter) != WRITE && offset + count > i_size) { 2866 if (offset >= i_size) 2867 return 0; 2868 iov_iter_truncate(iter, fuse_round_up(i_size - offset)); 2869 count = iov_iter_count(iter); 2870 } 2871 2872 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL); 2873 if (!io) 2874 return -ENOMEM; 2875 spin_lock_init(&io->lock); 2876 kref_init(&io->refcnt); 2877 io->reqs = 1; 2878 io->bytes = -1; 2879 io->size = 0; 2880 io->offset = offset; 2881 io->write = (iov_iter_rw(iter) == WRITE); 2882 io->err = 0; 2883 /* 2884 * By default, we want to optimize all I/Os with async request 2885 * submission to the client filesystem if supported. 2886 */ 2887 io->async = async_dio; 2888 io->iocb = iocb; 2889 io->blocking = is_sync_kiocb(iocb); 2890 2891 /* 2892 * We cannot asynchronously extend the size of a file. 2893 * In such case the aio will behave exactly like sync io. 2894 */ 2895 if ((offset + count > i_size) && iov_iter_rw(iter) == WRITE) 2896 io->blocking = true; 2897 2898 if (io->async && io->blocking) { 2899 /* 2900 * Additional reference to keep io around after 2901 * calling fuse_aio_complete() 2902 */ 2903 kref_get(&io->refcnt); 2904 io->done = &wait; 2905 } 2906 2907 if (iov_iter_rw(iter) == WRITE) { 2908 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); 2909 fuse_invalidate_attr(inode); 2910 } else { 2911 ret = __fuse_direct_read(io, iter, &pos); 2912 } 2913 2914 if (io->async) { 2915 fuse_aio_complete(io, ret < 0 ? ret : 0, -1); 2916 2917 /* we have a non-extending, async request, so return */ 2918 if (!io->blocking) 2919 return -EIOCBQUEUED; 2920 2921 wait_for_completion(&wait); 2922 ret = fuse_get_res_by_io(io); 2923 } 2924 2925 kref_put(&io->refcnt, fuse_io_release); 2926 2927 if (iov_iter_rw(iter) == WRITE) { 2928 if (ret > 0) 2929 fuse_write_update_size(inode, pos); 2930 else if (ret < 0 && offset + count > i_size) 2931 fuse_do_truncate(file); 2932 } 2933 2934 return ret; 2935 } 2936 2937 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, 2938 loff_t length) 2939 { 2940 struct fuse_file *ff = file->private_data; 2941 struct inode *inode = file_inode(file); 2942 struct fuse_inode *fi = get_fuse_inode(inode); 2943 struct fuse_conn *fc = ff->fc; 2944 FUSE_ARGS(args); 2945 struct fuse_fallocate_in inarg = { 2946 .fh = ff->fh, 2947 .offset = offset, 2948 .length = length, 2949 .mode = mode 2950 }; 2951 int err; 2952 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) || 2953 (mode & FALLOC_FL_PUNCH_HOLE); 2954 2955 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2956 return -EOPNOTSUPP; 2957 2958 if (fc->no_fallocate) 2959 return -EOPNOTSUPP; 2960 2961 if (lock_inode) { 2962 inode_lock(inode); 2963 if (mode & FALLOC_FL_PUNCH_HOLE) { 2964 loff_t endbyte = offset + length - 1; 2965 err = filemap_write_and_wait_range(inode->i_mapping, 2966 offset, endbyte); 2967 if (err) 2968 goto out; 2969 2970 fuse_sync_writes(inode); 2971 } 2972 } 2973 2974 if (!(mode & FALLOC_FL_KEEP_SIZE)) 2975 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 2976 2977 args.in.h.opcode = FUSE_FALLOCATE; 2978 args.in.h.nodeid = ff->nodeid; 2979 args.in.numargs = 1; 2980 args.in.args[0].size = sizeof(inarg); 2981 args.in.args[0].value = &inarg; 2982 err = fuse_simple_request(fc, &args); 2983 if (err == -ENOSYS) { 2984 fc->no_fallocate = 1; 2985 err = -EOPNOTSUPP; 2986 } 2987 if (err) 2988 goto out; 2989 2990 /* we could have extended the file */ 2991 if (!(mode & FALLOC_FL_KEEP_SIZE)) { 2992 bool changed = fuse_write_update_size(inode, offset + length); 2993 2994 if (changed && fc->writeback_cache) 2995 file_update_time(file); 2996 } 2997 2998 if (mode & FALLOC_FL_PUNCH_HOLE) 2999 truncate_pagecache_range(inode, offset, offset + length - 1); 3000 3001 fuse_invalidate_attr(inode); 3002 3003 out: 3004 if (!(mode & FALLOC_FL_KEEP_SIZE)) 3005 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 3006 3007 if (lock_inode) 3008 inode_unlock(inode); 3009 3010 return err; 3011 } 3012 3013 static const struct file_operations fuse_file_operations = { 3014 .llseek = fuse_file_llseek, 3015 .read_iter = fuse_file_read_iter, 3016 .write_iter = fuse_file_write_iter, 3017 .mmap = fuse_file_mmap, 3018 .open = fuse_open, 3019 .flush = fuse_flush, 3020 .release = fuse_release, 3021 .fsync = fuse_fsync, 3022 .lock = fuse_file_lock, 3023 .flock = fuse_file_flock, 3024 .splice_read = generic_file_splice_read, 3025 .unlocked_ioctl = fuse_file_ioctl, 3026 .compat_ioctl = fuse_file_compat_ioctl, 3027 .poll = fuse_file_poll, 3028 .fallocate = fuse_file_fallocate, 3029 }; 3030 3031 static const struct file_operations fuse_direct_io_file_operations = { 3032 .llseek = fuse_file_llseek, 3033 .read_iter = fuse_direct_read_iter, 3034 .write_iter = fuse_direct_write_iter, 3035 .mmap = fuse_direct_mmap, 3036 .open = fuse_open, 3037 .flush = fuse_flush, 3038 .release = fuse_release, 3039 .fsync = fuse_fsync, 3040 .lock = fuse_file_lock, 3041 .flock = fuse_file_flock, 3042 .unlocked_ioctl = fuse_file_ioctl, 3043 .compat_ioctl = fuse_file_compat_ioctl, 3044 .poll = fuse_file_poll, 3045 .fallocate = fuse_file_fallocate, 3046 /* no splice_read */ 3047 }; 3048 3049 static const struct address_space_operations fuse_file_aops = { 3050 .readpage = fuse_readpage, 3051 .writepage = fuse_writepage, 3052 .writepages = fuse_writepages, 3053 .launder_page = fuse_launder_page, 3054 .readpages = fuse_readpages, 3055 .set_page_dirty = __set_page_dirty_nobuffers, 3056 .bmap = fuse_bmap, 3057 .direct_IO = fuse_direct_IO, 3058 .write_begin = fuse_write_begin, 3059 .write_end = fuse_write_end, 3060 }; 3061 3062 void fuse_init_file_inode(struct inode *inode) 3063 { 3064 inode->i_fop = &fuse_file_operations; 3065 inode->i_data.a_ops = &fuse_file_aops; 3066 } 3067