1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 23d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 31d3576fdSSage Weil 41d3576fdSSage Weil #include <linux/backing-dev.h> 51d3576fdSSage Weil #include <linux/fs.h> 61d3576fdSSage Weil #include <linux/mm.h> 71d3576fdSSage Weil #include <linux/pagemap.h> 81d3576fdSSage Weil #include <linux/writeback.h> /* generic_writepages */ 95a0e3ad6STejun Heo #include <linux/slab.h> 101d3576fdSSage Weil #include <linux/pagevec.h> 111d3576fdSSage Weil #include <linux/task_io_accounting_ops.h> 12f361bf4aSIngo Molnar #include <linux/signal.h> 131d3576fdSSage Weil 141d3576fdSSage Weil #include "super.h" 153d14c5d2SYehuda Sadeh #include "mds_client.h" 1699ccbd22SMilosz Tanski #include "cache.h" 173d14c5d2SYehuda Sadeh #include <linux/ceph/osd_client.h> 181d3576fdSSage Weil 191d3576fdSSage Weil /* 201d3576fdSSage Weil * Ceph address space ops. 211d3576fdSSage Weil * 221d3576fdSSage Weil * There are a few funny things going on here. 231d3576fdSSage Weil * 241d3576fdSSage Weil * The page->private field is used to reference a struct 251d3576fdSSage Weil * ceph_snap_context for _every_ dirty page. This indicates which 261d3576fdSSage Weil * snapshot the page was logically dirtied in, and thus which snap 271d3576fdSSage Weil * context needs to be associated with the osd write during writeback. 281d3576fdSSage Weil * 291d3576fdSSage Weil * Similarly, struct ceph_inode_info maintains a set of counters to 3025985edcSLucas De Marchi * count dirty pages on the inode. In the absence of snapshots, 311d3576fdSSage Weil * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. 321d3576fdSSage Weil * 331d3576fdSSage Weil * When a snapshot is taken (that is, when the client receives 341d3576fdSSage Weil * notification that a snapshot was taken), each inode with caps and 351d3576fdSSage Weil * with dirty pages (dirty pages implies there is a cap) gets a new 361d3576fdSSage Weil * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending 371d3576fdSSage Weil * order, new snaps go to the tail). The i_wrbuffer_ref_head count is 381d3576fdSSage Weil * moved to capsnap->dirty. (Unless a sync write is currently in 391d3576fdSSage Weil * progress. In that case, the capsnap is said to be "pending", new 401d3576fdSSage Weil * writes cannot start, and the capsnap isn't "finalized" until the 411d3576fdSSage Weil * write completes (or fails) and a final size/mtime for the inode for 421d3576fdSSage Weil * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. 431d3576fdSSage Weil * 441d3576fdSSage Weil * On writeback, we must submit writes to the osd IN SNAP ORDER. So, 451d3576fdSSage Weil * we look for the first capsnap in i_cap_snaps and write out pages in 461d3576fdSSage Weil * that snap context _only_. Then we move on to the next capsnap, 471d3576fdSSage Weil * eventually reaching the "live" or "head" context (i.e., pages that 481d3576fdSSage Weil * are not yet snapped) and are writing the most recently dirtied 491d3576fdSSage Weil * pages. 501d3576fdSSage Weil * 511d3576fdSSage Weil * Invalidate and so forth must take care to ensure the dirty page 521d3576fdSSage Weil * accounting is preserved. 531d3576fdSSage Weil */ 541d3576fdSSage Weil 552baba250SYehuda Sadeh #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) 562baba250SYehuda Sadeh #define CONGESTION_OFF_THRESH(congestion_kb) \ 572baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) - \ 582baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) >> 2)) 592baba250SYehuda Sadeh 6061600ef8SYan, Zheng static inline struct ceph_snap_context *page_snap_context(struct page *page) 6161600ef8SYan, Zheng { 6261600ef8SYan, Zheng if (PagePrivate(page)) 6361600ef8SYan, Zheng return (void *)page->private; 6461600ef8SYan, Zheng return NULL; 6561600ef8SYan, Zheng } 661d3576fdSSage Weil 671d3576fdSSage Weil /* 681d3576fdSSage Weil * Dirty a page. Optimistically adjust accounting, on the assumption 691d3576fdSSage Weil * that we won't race with invalidate. If we do, readjust. 701d3576fdSSage Weil */ 711d3576fdSSage Weil static int ceph_set_page_dirty(struct page *page) 721d3576fdSSage Weil { 731d3576fdSSage Weil struct address_space *mapping = page->mapping; 741d3576fdSSage Weil struct inode *inode; 751d3576fdSSage Weil struct ceph_inode_info *ci; 761d3576fdSSage Weil struct ceph_snap_context *snapc; 777d6e1f54SSha Zhengju int ret; 781d3576fdSSage Weil 791d3576fdSSage Weil if (unlikely(!mapping)) 801d3576fdSSage Weil return !TestSetPageDirty(page); 811d3576fdSSage Weil 827d6e1f54SSha Zhengju if (PageDirty(page)) { 831d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu -- already dirty\n", 841d3576fdSSage Weil mapping->host, page, page->index); 857d6e1f54SSha Zhengju BUG_ON(!PagePrivate(page)); 861d3576fdSSage Weil return 0; 871d3576fdSSage Weil } 881d3576fdSSage Weil 891d3576fdSSage Weil inode = mapping->host; 901d3576fdSSage Weil ci = ceph_inode(inode); 911d3576fdSSage Weil 921d3576fdSSage Weil /* dirty the head */ 93be655596SSage Weil spin_lock(&ci->i_ceph_lock); 945dda377cSYan, Zheng BUG_ON(ci->i_wr_ref == 0); // caller should hold Fw reference 955dda377cSYan, Zheng if (__ceph_have_pending_cap_snap(ci)) { 965dda377cSYan, Zheng struct ceph_cap_snap *capsnap = 975dda377cSYan, Zheng list_last_entry(&ci->i_cap_snaps, 985dda377cSYan, Zheng struct ceph_cap_snap, 995dda377cSYan, Zheng ci_item); 1005dda377cSYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 1015dda377cSYan, Zheng capsnap->dirty_pages++; 1025dda377cSYan, Zheng } else { 1035dda377cSYan, Zheng BUG_ON(!ci->i_head_snapc); 1045dda377cSYan, Zheng snapc = ceph_get_snap_context(ci->i_head_snapc); 1051d3576fdSSage Weil ++ci->i_wrbuffer_ref_head; 1065dda377cSYan, Zheng } 1071d3576fdSSage Weil if (ci->i_wrbuffer_ref == 0) 1080444d76aSDave Chinner ihold(inode); 1091d3576fdSSage Weil ++ci->i_wrbuffer_ref; 1101d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu head %d/%d -> %d/%d " 1111d3576fdSSage Weil "snapc %p seq %lld (%d snaps)\n", 1121d3576fdSSage Weil mapping->host, page, page->index, 1131d3576fdSSage Weil ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, 1141d3576fdSSage Weil ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, 1151d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 116be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 1171d3576fdSSage Weil 1181d3576fdSSage Weil /* 1191d3576fdSSage Weil * Reference snap context in page->private. Also set 1201d3576fdSSage Weil * PagePrivate so that we get invalidatepage callback. 1211d3576fdSSage Weil */ 1227d6e1f54SSha Zhengju BUG_ON(PagePrivate(page)); 1231d3576fdSSage Weil page->private = (unsigned long)snapc; 1241d3576fdSSage Weil SetPagePrivate(page); 1251d3576fdSSage Weil 1267d6e1f54SSha Zhengju ret = __set_page_dirty_nobuffers(page); 1277d6e1f54SSha Zhengju WARN_ON(!PageLocked(page)); 1287d6e1f54SSha Zhengju WARN_ON(!page->mapping); 1291d3576fdSSage Weil 1307d6e1f54SSha Zhengju return ret; 1311d3576fdSSage Weil } 1321d3576fdSSage Weil 1331d3576fdSSage Weil /* 1341d3576fdSSage Weil * If we are truncating the full page (i.e. offset == 0), adjust the 1351d3576fdSSage Weil * dirty page counters appropriately. Only called if there is private 1361d3576fdSSage Weil * data on the page. 1371d3576fdSSage Weil */ 138d47992f8SLukas Czerner static void ceph_invalidatepage(struct page *page, unsigned int offset, 139d47992f8SLukas Czerner unsigned int length) 1401d3576fdSSage Weil { 1414ce1e9adSAlexander Beregalov struct inode *inode; 1421d3576fdSSage Weil struct ceph_inode_info *ci; 14361600ef8SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 1441d3576fdSSage Weil 1454ce1e9adSAlexander Beregalov inode = page->mapping->host; 146b150f5c1SMilosz Tanski ci = ceph_inode(inode); 147b150f5c1SMilosz Tanski 14809cbfeafSKirill A. Shutemov if (offset != 0 || length != PAGE_SIZE) { 149b150f5c1SMilosz Tanski dout("%p invalidatepage %p idx %lu partial dirty page %u~%u\n", 150b150f5c1SMilosz Tanski inode, page, page->index, offset, length); 151b150f5c1SMilosz Tanski return; 152b150f5c1SMilosz Tanski } 1534ce1e9adSAlexander Beregalov 15499ccbd22SMilosz Tanski ceph_invalidate_fscache_page(inode, page); 15599ccbd22SMilosz Tanski 156b072d774SYan, Zheng WARN_ON(!PageLocked(page)); 15799ccbd22SMilosz Tanski if (!PagePrivate(page)) 15899ccbd22SMilosz Tanski return; 15999ccbd22SMilosz Tanski 1601d3576fdSSage Weil ClearPageChecked(page); 1611d3576fdSSage Weil 162569d39fcSLukas Czerner dout("%p invalidatepage %p idx %lu full dirty page\n", 163569d39fcSLukas Czerner inode, page, page->index); 164b150f5c1SMilosz Tanski 1651d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 1661d3576fdSSage Weil ceph_put_snap_context(snapc); 1671d3576fdSSage Weil page->private = 0; 1681d3576fdSSage Weil ClearPagePrivate(page); 1691d3576fdSSage Weil } 1701d3576fdSSage Weil 1711d3576fdSSage Weil static int ceph_releasepage(struct page *page, gfp_t g) 1721d3576fdSSage Weil { 173e55f1a18SNeilBrown dout("%p releasepage %p idx %lu (%sdirty)\n", page->mapping->host, 174e55f1a18SNeilBrown page, page->index, PageDirty(page) ? "" : "not "); 17599ccbd22SMilosz Tanski 17699ccbd22SMilosz Tanski /* Can we release the page from the cache? */ 17799ccbd22SMilosz Tanski if (!ceph_release_fscache_page(page, g)) 1781d3576fdSSage Weil return 0; 17999ccbd22SMilosz Tanski 18099ccbd22SMilosz Tanski return !PagePrivate(page); 1811d3576fdSSage Weil } 1821d3576fdSSage Weil 1831d3576fdSSage Weil /* 1841d3576fdSSage Weil * read a single page, without unlocking it. 1851d3576fdSSage Weil */ 186dd2bc473SYan, Zheng static int ceph_do_readpage(struct file *filp, struct page *page) 1871d3576fdSSage Weil { 188496ad9aaSAl Viro struct inode *inode = file_inode(filp); 1891d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 1903d14c5d2SYehuda Sadeh struct ceph_osd_client *osdc = 1913d14c5d2SYehuda Sadeh &ceph_inode_to_client(inode)->client->osdc; 1921d3576fdSSage Weil int err = 0; 19383701246SYan, Zheng u64 off = page_offset(page); 19409cbfeafSKirill A. Shutemov u64 len = PAGE_SIZE; 1951d3576fdSSage Weil 19683701246SYan, Zheng if (off >= i_size_read(inode)) { 19709cbfeafSKirill A. Shutemov zero_user_segment(page, 0, PAGE_SIZE); 19883701246SYan, Zheng SetPageUptodate(page); 19983701246SYan, Zheng return 0; 20083701246SYan, Zheng } 20199ccbd22SMilosz Tanski 202fcc02d2aSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 20383701246SYan, Zheng /* 204fcc02d2aSYan, Zheng * Uptodate inline data should have been added 205fcc02d2aSYan, Zheng * into page cache while getting Fcr caps. 20683701246SYan, Zheng */ 207fcc02d2aSYan, Zheng if (off == 0) 20883701246SYan, Zheng return -EINVAL; 20909cbfeafSKirill A. Shutemov zero_user_segment(page, 0, PAGE_SIZE); 210fcc02d2aSYan, Zheng SetPageUptodate(page); 211fcc02d2aSYan, Zheng return 0; 212fcc02d2aSYan, Zheng } 21383701246SYan, Zheng 21483701246SYan, Zheng err = ceph_readpage_from_fscache(inode, page); 21599ccbd22SMilosz Tanski if (err == 0) 216dd2bc473SYan, Zheng return -EINPROGRESS; 21799ccbd22SMilosz Tanski 2181d3576fdSSage Weil dout("readpage inode %p file %p page %p index %lu\n", 2191d3576fdSSage Weil inode, filp, page, page->index); 2201d3576fdSSage Weil err = ceph_osdc_readpages(osdc, ceph_vino(inode), &ci->i_layout, 22183701246SYan, Zheng off, &len, 2221d3576fdSSage Weil ci->i_truncate_seq, ci->i_truncate_size, 223b7495fc2SSage Weil &page, 1, 0); 2241d3576fdSSage Weil if (err == -ENOENT) 2251d3576fdSSage Weil err = 0; 2261d3576fdSSage Weil if (err < 0) { 2271d3576fdSSage Weil SetPageError(page); 22818302805SLi Wang ceph_fscache_readpage_cancel(inode, page); 2291d3576fdSSage Weil goto out; 23023cd573bSZhang Zhen } 23109cbfeafSKirill A. Shutemov if (err < PAGE_SIZE) 2321d3576fdSSage Weil /* zero fill remainder of page */ 23309cbfeafSKirill A. Shutemov zero_user_segment(page, err, PAGE_SIZE); 23423cd573bSZhang Zhen else 23556f91aadSLi Wang flush_dcache_page(page); 2361d3576fdSSage Weil 23723cd573bSZhang Zhen SetPageUptodate(page); 23899ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 23999ccbd22SMilosz Tanski 2401d3576fdSSage Weil out: 2411d3576fdSSage Weil return err < 0 ? err : 0; 2421d3576fdSSage Weil } 2431d3576fdSSage Weil 2441d3576fdSSage Weil static int ceph_readpage(struct file *filp, struct page *page) 2451d3576fdSSage Weil { 246dd2bc473SYan, Zheng int r = ceph_do_readpage(filp, page); 247dd2bc473SYan, Zheng if (r != -EINPROGRESS) 2481d3576fdSSage Weil unlock_page(page); 249dd2bc473SYan, Zheng else 250dd2bc473SYan, Zheng r = 0; 2511d3576fdSSage Weil return r; 2521d3576fdSSage Weil } 2531d3576fdSSage Weil 2541d3576fdSSage Weil /* 2557c272194SSage Weil * Finish an async read(ahead) op. 2561d3576fdSSage Weil */ 25785e084feSIlya Dryomov static void finish_read(struct ceph_osd_request *req) 2581d3576fdSSage Weil { 2597c272194SSage Weil struct inode *inode = req->r_inode; 26087060c10SAlex Elder struct ceph_osd_data *osd_data; 26185e084feSIlya Dryomov int rc = req->r_result <= 0 ? req->r_result : 0; 26285e084feSIlya Dryomov int bytes = req->r_result >= 0 ? req->r_result : 0; 263e0c59487SAlex Elder int num_pages; 2647c272194SSage Weil int i; 2657c272194SSage Weil 2667c272194SSage Weil dout("finish_read %p req %p rc %d bytes %d\n", inode, req, rc, bytes); 2677c272194SSage Weil 2687c272194SSage Weil /* unlock all pages, zeroing any data we didn't read */ 269406e2c9fSAlex Elder osd_data = osd_req_op_extent_osd_data(req, 0); 27087060c10SAlex Elder BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 27187060c10SAlex Elder num_pages = calc_pages_for((u64)osd_data->alignment, 27287060c10SAlex Elder (u64)osd_data->length); 273e0c59487SAlex Elder for (i = 0; i < num_pages; i++) { 27487060c10SAlex Elder struct page *page = osd_data->pages[i]; 2757c272194SSage Weil 276368e3585SYan, Zheng if (rc < 0 && rc != -ENOENT) { 277368e3585SYan, Zheng ceph_fscache_readpage_cancel(inode, page); 278f36132a7SLi Wang goto unlock; 279368e3585SYan, Zheng } 28009cbfeafSKirill A. Shutemov if (bytes < (int)PAGE_SIZE) { 2817c272194SSage Weil /* zero (remainder of) page */ 2827c272194SSage Weil int s = bytes < 0 ? 0 : bytes; 28309cbfeafSKirill A. Shutemov zero_user_segment(page, s, PAGE_SIZE); 2847c272194SSage Weil } 2857c272194SSage Weil dout("finish_read %p uptodate %p idx %lu\n", inode, page, 2867c272194SSage Weil page->index); 2877c272194SSage Weil flush_dcache_page(page); 2887c272194SSage Weil SetPageUptodate(page); 28999ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 290f36132a7SLi Wang unlock: 2917c272194SSage Weil unlock_page(page); 29209cbfeafSKirill A. Shutemov put_page(page); 29309cbfeafSKirill A. Shutemov bytes -= PAGE_SIZE; 2947c272194SSage Weil } 29587060c10SAlex Elder kfree(osd_data->pages); 2967c272194SSage Weil } 2977c272194SSage Weil 2987c272194SSage Weil /* 2997c272194SSage Weil * start an async read(ahead) operation. return nr_pages we submitted 3007c272194SSage Weil * a read for on success, or negative error code. 3017c272194SSage Weil */ 3020d66a487SSage Weil static int start_read(struct inode *inode, struct list_head *page_list, int max) 3037c272194SSage Weil { 3047c272194SSage Weil struct ceph_osd_client *osdc = 3057c272194SSage Weil &ceph_inode_to_client(inode)->client->osdc; 3067c272194SSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 3077c272194SSage Weil struct page *page = list_entry(page_list->prev, struct page, lru); 308acead002SAlex Elder struct ceph_vino vino; 3097c272194SSage Weil struct ceph_osd_request *req; 3107c272194SSage Weil u64 off; 3117c272194SSage Weil u64 len; 3127c272194SSage Weil int i; 3131d3576fdSSage Weil struct page **pages; 3147c272194SSage Weil pgoff_t next_index; 3157c272194SSage Weil int nr_pages = 0; 3162b1ac852SYan, Zheng int got = 0; 3172b1ac852SYan, Zheng int ret = 0; 3182b1ac852SYan, Zheng 3192b1ac852SYan, Zheng if (!current->journal_info) { 3202b1ac852SYan, Zheng /* caller of readpages does not hold buffer and read caps 3212b1ac852SYan, Zheng * (fadvise, madvise and readahead cases) */ 3222b1ac852SYan, Zheng int want = CEPH_CAP_FILE_CACHE; 3232b1ac852SYan, Zheng ret = ceph_try_get_caps(ci, CEPH_CAP_FILE_RD, want, &got); 3242b1ac852SYan, Zheng if (ret < 0) { 3252b1ac852SYan, Zheng dout("start_read %p, error getting cap\n", inode); 3262b1ac852SYan, Zheng } else if (!(got & want)) { 3272b1ac852SYan, Zheng dout("start_read %p, no cache cap\n", inode); 3282b1ac852SYan, Zheng ret = 0; 3292b1ac852SYan, Zheng } 3302b1ac852SYan, Zheng if (ret <= 0) { 3312b1ac852SYan, Zheng if (got) 3322b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 3332b1ac852SYan, Zheng while (!list_empty(page_list)) { 3342b1ac852SYan, Zheng page = list_entry(page_list->prev, 3352b1ac852SYan, Zheng struct page, lru); 3362b1ac852SYan, Zheng list_del(&page->lru); 3372b1ac852SYan, Zheng put_page(page); 3382b1ac852SYan, Zheng } 3392b1ac852SYan, Zheng return ret; 3402b1ac852SYan, Zheng } 3412b1ac852SYan, Zheng } 3427c272194SSage Weil 3436285bc23SAlex Elder off = (u64) page_offset(page); 3447c272194SSage Weil 3457c272194SSage Weil /* count pages */ 3467c272194SSage Weil next_index = page->index; 3477c272194SSage Weil list_for_each_entry_reverse(page, page_list, lru) { 3487c272194SSage Weil if (page->index != next_index) 3497c272194SSage Weil break; 3507c272194SSage Weil nr_pages++; 3517c272194SSage Weil next_index++; 3520d66a487SSage Weil if (max && nr_pages == max) 3530d66a487SSage Weil break; 3547c272194SSage Weil } 35509cbfeafSKirill A. Shutemov len = nr_pages << PAGE_SHIFT; 3567c272194SSage Weil dout("start_read %p nr_pages %d is %lld~%lld\n", inode, nr_pages, 3577c272194SSage Weil off, len); 358acead002SAlex Elder vino = ceph_vino(inode); 359acead002SAlex Elder req = ceph_osdc_new_request(osdc, &ci->i_layout, vino, off, &len, 360715e4cd4SYan, Zheng 0, 1, CEPH_OSD_OP_READ, 361acead002SAlex Elder CEPH_OSD_FLAG_READ, NULL, 3627c272194SSage Weil ci->i_truncate_seq, ci->i_truncate_size, 363acead002SAlex Elder false); 3642b1ac852SYan, Zheng if (IS_ERR(req)) { 3652b1ac852SYan, Zheng ret = PTR_ERR(req); 3662b1ac852SYan, Zheng goto out; 3672b1ac852SYan, Zheng } 3681d3576fdSSage Weil 3691d3576fdSSage Weil /* build page vector */ 370cf7b7e14SAlex Elder nr_pages = calc_pages_for(0, len); 371687265e5SYan, Zheng pages = kmalloc(sizeof(*pages) * nr_pages, GFP_KERNEL); 3722b1ac852SYan, Zheng if (!pages) { 3737c272194SSage Weil ret = -ENOMEM; 3742b1ac852SYan, Zheng goto out_put; 3752b1ac852SYan, Zheng } 3767c272194SSage Weil for (i = 0; i < nr_pages; ++i) { 3777c272194SSage Weil page = list_entry(page_list->prev, struct page, lru); 3787c272194SSage Weil BUG_ON(PageLocked(page)); 3797c272194SSage Weil list_del(&page->lru); 3801d3576fdSSage Weil 3817c272194SSage Weil dout("start_read %p adding %p idx %lu\n", inode, page, 3827c272194SSage Weil page->index); 3837c272194SSage Weil if (add_to_page_cache_lru(page, &inode->i_data, page->index, 384687265e5SYan, Zheng GFP_KERNEL)) { 385d4d3aa38SMilosz Tanski ceph_fscache_uncache_page(inode, page); 38609cbfeafSKirill A. Shutemov put_page(page); 3877c272194SSage Weil dout("start_read %p add_to_page_cache failed %p\n", 3887c272194SSage Weil inode, page); 3897c272194SSage Weil nr_pages = i; 3901afe4785SYan, Zheng if (nr_pages > 0) { 3911afe4785SYan, Zheng len = nr_pages << PAGE_SHIFT; 392d641df81SYan, Zheng osd_req_op_extent_update(req, 0, len); 3931afe4785SYan, Zheng break; 3941afe4785SYan, Zheng } 3957c272194SSage Weil goto out_pages; 3961d3576fdSSage Weil } 3977c272194SSage Weil pages[i] = page; 3981d3576fdSSage Weil } 399406e2c9fSAlex Elder osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false); 4007c272194SSage Weil req->r_callback = finish_read; 4017c272194SSage Weil req->r_inode = inode; 4027c272194SSage Weil 4037c272194SSage Weil dout("start_read %p starting %p %lld~%lld\n", inode, req, off, len); 4047c272194SSage Weil ret = ceph_osdc_start_request(osdc, req, false); 4057c272194SSage Weil if (ret < 0) 4067c272194SSage Weil goto out_pages; 4077c272194SSage Weil ceph_osdc_put_request(req); 4082b1ac852SYan, Zheng 4092b1ac852SYan, Zheng /* After adding locked pages to page cache, the inode holds cache cap. 4102b1ac852SYan, Zheng * So we can drop our cap refs. */ 4112b1ac852SYan, Zheng if (got) 4122b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 4132b1ac852SYan, Zheng 4147c272194SSage Weil return nr_pages; 4157c272194SSage Weil 4167c272194SSage Weil out_pages: 4171afe4785SYan, Zheng for (i = 0; i < nr_pages; ++i) { 4181afe4785SYan, Zheng ceph_fscache_readpage_cancel(inode, pages[i]); 4191afe4785SYan, Zheng unlock_page(pages[i]); 4201afe4785SYan, Zheng } 4211afe4785SYan, Zheng ceph_put_page_vector(pages, nr_pages, false); 4222b1ac852SYan, Zheng out_put: 4237c272194SSage Weil ceph_osdc_put_request(req); 4242b1ac852SYan, Zheng out: 4252b1ac852SYan, Zheng if (got) 4262b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 4277c272194SSage Weil return ret; 4281d3576fdSSage Weil } 4291d3576fdSSage Weil 4307c272194SSage Weil 4311d3576fdSSage Weil /* 4321d3576fdSSage Weil * Read multiple pages. Leave pages we don't read + unlock in page_list; 4331d3576fdSSage Weil * the caller (VM) cleans them up. 4341d3576fdSSage Weil */ 4351d3576fdSSage Weil static int ceph_readpages(struct file *file, struct address_space *mapping, 4361d3576fdSSage Weil struct list_head *page_list, unsigned nr_pages) 4371d3576fdSSage Weil { 438496ad9aaSAl Viro struct inode *inode = file_inode(file); 4390d66a487SSage Weil struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 4401d3576fdSSage Weil int rc = 0; 4410d66a487SSage Weil int max = 0; 4421d3576fdSSage Weil 44383701246SYan, Zheng if (ceph_inode(inode)->i_inline_version != CEPH_INLINE_NONE) 44483701246SYan, Zheng return -EINVAL; 44583701246SYan, Zheng 44699ccbd22SMilosz Tanski rc = ceph_readpages_from_fscache(mapping->host, mapping, page_list, 44799ccbd22SMilosz Tanski &nr_pages); 44899ccbd22SMilosz Tanski 44999ccbd22SMilosz Tanski if (rc == 0) 45099ccbd22SMilosz Tanski goto out; 45199ccbd22SMilosz Tanski 452aa187926SYan, Zheng max = fsc->mount_options->rsize >> PAGE_SHIFT; 453aa187926SYan, Zheng dout("readpages %p file %p nr_pages %d max %d\n", 454aa187926SYan, Zheng inode, file, nr_pages, max); 4557c272194SSage Weil while (!list_empty(page_list)) { 4560d66a487SSage Weil rc = start_read(inode, page_list, max); 4571d3576fdSSage Weil if (rc < 0) 4581d3576fdSSage Weil goto out; 4591d3576fdSSage Weil } 4601d3576fdSSage Weil out: 46176be778bSMilosz Tanski ceph_fscache_readpages_cancel(inode, page_list); 46276be778bSMilosz Tanski 4637c272194SSage Weil dout("readpages %p file %p ret %d\n", inode, file, rc); 4641d3576fdSSage Weil return rc; 4651d3576fdSSage Weil } 4661d3576fdSSage Weil 4671f934b00SYan, Zheng struct ceph_writeback_ctl 4681f934b00SYan, Zheng { 4691f934b00SYan, Zheng loff_t i_size; 4701f934b00SYan, Zheng u64 truncate_size; 4711f934b00SYan, Zheng u32 truncate_seq; 4721f934b00SYan, Zheng bool size_stable; 4732a2d927eSYan, Zheng bool head_snapc; 4741f934b00SYan, Zheng }; 4751f934b00SYan, Zheng 4761d3576fdSSage Weil /* 4771d3576fdSSage Weil * Get ref for the oldest snapc for an inode with dirty data... that is, the 4781d3576fdSSage Weil * only snap context we are allowed to write back. 4791d3576fdSSage Weil */ 4801f934b00SYan, Zheng static struct ceph_snap_context * 48105455e11SYan, Zheng get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, 48205455e11SYan, Zheng struct ceph_snap_context *page_snapc) 4831d3576fdSSage Weil { 4841d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 4851d3576fdSSage Weil struct ceph_snap_context *snapc = NULL; 4861d3576fdSSage Weil struct ceph_cap_snap *capsnap = NULL; 4871d3576fdSSage Weil 488be655596SSage Weil spin_lock(&ci->i_ceph_lock); 4891d3576fdSSage Weil list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 4901d3576fdSSage Weil dout(" cap_snap %p snapc %p has %d dirty pages\n", capsnap, 4911d3576fdSSage Weil capsnap->context, capsnap->dirty_pages); 49205455e11SYan, Zheng if (!capsnap->dirty_pages) 49305455e11SYan, Zheng continue; 49405455e11SYan, Zheng 49505455e11SYan, Zheng /* get i_size, truncate_{seq,size} for page_snapc? */ 49605455e11SYan, Zheng if (snapc && capsnap->context != page_snapc) 49705455e11SYan, Zheng continue; 49805455e11SYan, Zheng 4991f934b00SYan, Zheng if (ctl) { 5001f934b00SYan, Zheng if (capsnap->writing) { 5011f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 5021f934b00SYan, Zheng ctl->size_stable = false; 5031f934b00SYan, Zheng } else { 5041f934b00SYan, Zheng ctl->i_size = capsnap->size; 5051f934b00SYan, Zheng ctl->size_stable = true; 5061f934b00SYan, Zheng } 5071f934b00SYan, Zheng ctl->truncate_size = capsnap->truncate_size; 5081f934b00SYan, Zheng ctl->truncate_seq = capsnap->truncate_seq; 5092a2d927eSYan, Zheng ctl->head_snapc = false; 5101f934b00SYan, Zheng } 51105455e11SYan, Zheng 51205455e11SYan, Zheng if (snapc) 5131d3576fdSSage Weil break; 51405455e11SYan, Zheng 51505455e11SYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 51605455e11SYan, Zheng if (!page_snapc || 51705455e11SYan, Zheng page_snapc == snapc || 51805455e11SYan, Zheng page_snapc->seq > snapc->seq) 51905455e11SYan, Zheng break; 5201d3576fdSSage Weil } 5217d8cb26dSSage Weil if (!snapc && ci->i_wrbuffer_ref_head) { 52280e755feSSage Weil snapc = ceph_get_snap_context(ci->i_head_snapc); 5231d3576fdSSage Weil dout(" head snapc %p has %d dirty pages\n", 5241d3576fdSSage Weil snapc, ci->i_wrbuffer_ref_head); 5251f934b00SYan, Zheng if (ctl) { 5261f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 5271f934b00SYan, Zheng ctl->truncate_size = ci->i_truncate_size; 5281f934b00SYan, Zheng ctl->truncate_seq = ci->i_truncate_seq; 5291f934b00SYan, Zheng ctl->size_stable = false; 5302a2d927eSYan, Zheng ctl->head_snapc = true; 5311f934b00SYan, Zheng } 5321d3576fdSSage Weil } 533be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 5341d3576fdSSage Weil return snapc; 5351d3576fdSSage Weil } 5361d3576fdSSage Weil 5371f934b00SYan, Zheng static u64 get_writepages_data_length(struct inode *inode, 5381f934b00SYan, Zheng struct page *page, u64 start) 5391f934b00SYan, Zheng { 5401f934b00SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 5411f934b00SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 5421f934b00SYan, Zheng struct ceph_cap_snap *capsnap = NULL; 5431f934b00SYan, Zheng u64 end = i_size_read(inode); 5441f934b00SYan, Zheng 5451f934b00SYan, Zheng if (snapc != ci->i_head_snapc) { 5461f934b00SYan, Zheng bool found = false; 5471f934b00SYan, Zheng spin_lock(&ci->i_ceph_lock); 5481f934b00SYan, Zheng list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 5491f934b00SYan, Zheng if (capsnap->context == snapc) { 5501f934b00SYan, Zheng if (!capsnap->writing) 5511f934b00SYan, Zheng end = capsnap->size; 5521f934b00SYan, Zheng found = true; 5531f934b00SYan, Zheng break; 5541f934b00SYan, Zheng } 5551f934b00SYan, Zheng } 5561f934b00SYan, Zheng spin_unlock(&ci->i_ceph_lock); 5571f934b00SYan, Zheng WARN_ON(!found); 5581f934b00SYan, Zheng } 5591f934b00SYan, Zheng if (end > page_offset(page) + PAGE_SIZE) 5601f934b00SYan, Zheng end = page_offset(page) + PAGE_SIZE; 5611f934b00SYan, Zheng return end > start ? end - start : 0; 5621f934b00SYan, Zheng } 5631f934b00SYan, Zheng 5641d3576fdSSage Weil /* 5651d3576fdSSage Weil * Write a single page, but leave the page locked. 5661d3576fdSSage Weil * 5671d3576fdSSage Weil * If we get a write error, set the page error bit, but still adjust the 5681d3576fdSSage Weil * dirty page accounting (i.e., page is no longer dirty). 5691d3576fdSSage Weil */ 5701d3576fdSSage Weil static int writepage_nounlock(struct page *page, struct writeback_control *wbc) 5711d3576fdSSage Weil { 5721d3576fdSSage Weil struct inode *inode; 5731d3576fdSSage Weil struct ceph_inode_info *ci; 5743d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc; 5756298a337SSage Weil struct ceph_snap_context *snapc, *oldest; 576fc2744aaSYan, Zheng loff_t page_off = page_offset(page); 5772baba250SYehuda Sadeh long writeback_stat; 57843986881SYan, Zheng int err, len = PAGE_SIZE; 5791f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 5801d3576fdSSage Weil 5811d3576fdSSage Weil dout("writepage %p idx %lu\n", page, page->index); 5821d3576fdSSage Weil 5831d3576fdSSage Weil inode = page->mapping->host; 5841d3576fdSSage Weil ci = ceph_inode(inode); 5853d14c5d2SYehuda Sadeh fsc = ceph_inode_to_client(inode); 5861d3576fdSSage Weil 5871d3576fdSSage Weil /* verify this is a writeable snap context */ 58861600ef8SYan, Zheng snapc = page_snap_context(page); 589d37b1d99SMarkus Elfring if (!snapc) { 5901d3576fdSSage Weil dout("writepage %p page %p not dirty?\n", inode, page); 59143986881SYan, Zheng return 0; 5921d3576fdSSage Weil } 59305455e11SYan, Zheng oldest = get_oldest_context(inode, &ceph_wbc, snapc); 5946298a337SSage Weil if (snapc->seq > oldest->seq) { 5951d3576fdSSage Weil dout("writepage %p page %p snapc %p not writeable - noop\n", 59661600ef8SYan, Zheng inode, page, snapc); 5971d3576fdSSage Weil /* we should only noop if called by kswapd */ 598fa71fefbSYan, Zheng WARN_ON(!(current->flags & PF_MEMALLOC)); 5996298a337SSage Weil ceph_put_snap_context(oldest); 600fa71fefbSYan, Zheng redirty_page_for_writepage(wbc, page); 60143986881SYan, Zheng return 0; 6021d3576fdSSage Weil } 6036298a337SSage Weil ceph_put_snap_context(oldest); 6041d3576fdSSage Weil 6051d3576fdSSage Weil /* is this a partial page at end of file? */ 6061f934b00SYan, Zheng if (page_off >= ceph_wbc.i_size) { 6071f934b00SYan, Zheng dout("%p page eof %llu\n", page, ceph_wbc.i_size); 60805455e11SYan, Zheng page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE); 60943986881SYan, Zheng return 0; 610fc2744aaSYan, Zheng } 61143986881SYan, Zheng 6121f934b00SYan, Zheng if (ceph_wbc.i_size < page_off + len) 6131f934b00SYan, Zheng len = ceph_wbc.i_size - page_off; 6141d3576fdSSage Weil 6151c0a9c2dSYan, Zheng dout("writepage %p page %p index %lu on %llu~%u snapc %p seq %lld\n", 6161c0a9c2dSYan, Zheng inode, page, page->index, page_off, len, snapc, snapc->seq); 6171d3576fdSSage Weil 6183d14c5d2SYehuda Sadeh writeback_stat = atomic_long_inc_return(&fsc->writeback_count); 6192baba250SYehuda Sadeh if (writeback_stat > 6203d14c5d2SYehuda Sadeh CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) 62109dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 6222baba250SYehuda Sadeh 6231d3576fdSSage Weil set_page_writeback(page); 6241f934b00SYan, Zheng err = ceph_osdc_writepages(&fsc->client->osdc, ceph_vino(inode), 6251f934b00SYan, Zheng &ci->i_layout, snapc, page_off, len, 6261f934b00SYan, Zheng ceph_wbc.truncate_seq, 6271f934b00SYan, Zheng ceph_wbc.truncate_size, 62824808826SAlex Elder &inode->i_mtime, &page, 1); 6291d3576fdSSage Weil if (err < 0) { 630ad15ec06SYan, Zheng struct writeback_control tmp_wbc; 631ad15ec06SYan, Zheng if (!wbc) 632ad15ec06SYan, Zheng wbc = &tmp_wbc; 633ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 634ad15ec06SYan, Zheng /* killed by SIGKILL */ 635ad15ec06SYan, Zheng dout("writepage interrupted page %p\n", page); 636ad15ec06SYan, Zheng redirty_page_for_writepage(wbc, page); 637ad15ec06SYan, Zheng end_page_writeback(page); 63843986881SYan, Zheng return err; 639ad15ec06SYan, Zheng } 640ad15ec06SYan, Zheng dout("writepage setting page/mapping error %d %p\n", 641ad15ec06SYan, Zheng err, page); 6421d3576fdSSage Weil SetPageError(page); 6431d3576fdSSage Weil mapping_set_error(&inode->i_data, err); 6441d3576fdSSage Weil wbc->pages_skipped++; 6451d3576fdSSage Weil } else { 6461d3576fdSSage Weil dout("writepage cleaned page %p\n", page); 6471d3576fdSSage Weil err = 0; /* vfs expects us to return 0 */ 6481d3576fdSSage Weil } 6491d3576fdSSage Weil page->private = 0; 6501d3576fdSSage Weil ClearPagePrivate(page); 6511d3576fdSSage Weil end_page_writeback(page); 6521d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 6536298a337SSage Weil ceph_put_snap_context(snapc); /* page's reference */ 6541d3576fdSSage Weil return err; 6551d3576fdSSage Weil } 6561d3576fdSSage Weil 6571d3576fdSSage Weil static int ceph_writepage(struct page *page, struct writeback_control *wbc) 6581d3576fdSSage Weil { 659dbd646a8SYehuda Sadeh int err; 660dbd646a8SYehuda Sadeh struct inode *inode = page->mapping->host; 661dbd646a8SYehuda Sadeh BUG_ON(!inode); 66270b666c3SSage Weil ihold(inode); 663dbd646a8SYehuda Sadeh err = writepage_nounlock(page, wbc); 664ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 665ad15ec06SYan, Zheng /* direct memory reclaimer was killed by SIGKILL. return 0 666ad15ec06SYan, Zheng * to prevent caller from setting mapping/page error */ 667ad15ec06SYan, Zheng err = 0; 668ad15ec06SYan, Zheng } 6691d3576fdSSage Weil unlock_page(page); 670dbd646a8SYehuda Sadeh iput(inode); 6711d3576fdSSage Weil return err; 6721d3576fdSSage Weil } 6731d3576fdSSage Weil 6741d3576fdSSage Weil /* 6751d3576fdSSage Weil * lame release_pages helper. release_pages() isn't exported to 6761d3576fdSSage Weil * modules. 6771d3576fdSSage Weil */ 6781d3576fdSSage Weil static void ceph_release_pages(struct page **pages, int num) 6791d3576fdSSage Weil { 6801d3576fdSSage Weil struct pagevec pvec; 6811d3576fdSSage Weil int i; 6821d3576fdSSage Weil 6831d3576fdSSage Weil pagevec_init(&pvec, 0); 6841d3576fdSSage Weil for (i = 0; i < num; i++) { 6851d3576fdSSage Weil if (pagevec_add(&pvec, pages[i]) == 0) 6861d3576fdSSage Weil pagevec_release(&pvec); 6871d3576fdSSage Weil } 6881d3576fdSSage Weil pagevec_release(&pvec); 6891d3576fdSSage Weil } 6901d3576fdSSage Weil 6911d3576fdSSage Weil /* 6921d3576fdSSage Weil * async writeback completion handler. 6931d3576fdSSage Weil * 6941d3576fdSSage Weil * If we get an error, set the mapping error bit, but not the individual 6951d3576fdSSage Weil * page error bits. 6961d3576fdSSage Weil */ 69785e084feSIlya Dryomov static void writepages_finish(struct ceph_osd_request *req) 6981d3576fdSSage Weil { 6991d3576fdSSage Weil struct inode *inode = req->r_inode; 7001d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 70187060c10SAlex Elder struct ceph_osd_data *osd_data; 7021d3576fdSSage Weil struct page *page; 7035b64640cSYan, Zheng int num_pages, total_pages = 0; 7045b64640cSYan, Zheng int i, j; 7055b64640cSYan, Zheng int rc = req->r_result; 7061d3576fdSSage Weil struct ceph_snap_context *snapc = req->r_snapc; 7071d3576fdSSage Weil struct address_space *mapping = inode->i_mapping; 7083d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 7095b64640cSYan, Zheng bool remove_page; 7101d3576fdSSage Weil 7115b64640cSYan, Zheng dout("writepages_finish %p rc %d\n", inode, rc); 71226544c62SJeff Layton if (rc < 0) { 7131d3576fdSSage Weil mapping_set_error(mapping, rc); 71426544c62SJeff Layton ceph_set_error_write(ci); 71526544c62SJeff Layton } else { 71626544c62SJeff Layton ceph_clear_error_write(ci); 71726544c62SJeff Layton } 718e63dc5c7SYehuda Sadeh 719e63dc5c7SYehuda Sadeh /* 720e63dc5c7SYehuda Sadeh * We lost the cache cap, need to truncate the page before 721e63dc5c7SYehuda Sadeh * it is unlocked, otherwise we'd truncate it later in the 722e63dc5c7SYehuda Sadeh * page truncation thread, possibly losing some data that 723e63dc5c7SYehuda Sadeh * raced its way in 724e63dc5c7SYehuda Sadeh */ 7255b64640cSYan, Zheng remove_page = !(ceph_caps_issued(ci) & 7265b64640cSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); 7275b64640cSYan, Zheng 7285b64640cSYan, Zheng /* clean all pages */ 7295b64640cSYan, Zheng for (i = 0; i < req->r_num_ops; i++) { 7305b64640cSYan, Zheng if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) 7315b64640cSYan, Zheng break; 7325b64640cSYan, Zheng 7335b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, i); 7345b64640cSYan, Zheng BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 7355b64640cSYan, Zheng num_pages = calc_pages_for((u64)osd_data->alignment, 7365b64640cSYan, Zheng (u64)osd_data->length); 7375b64640cSYan, Zheng total_pages += num_pages; 7385b64640cSYan, Zheng for (j = 0; j < num_pages; j++) { 7395b64640cSYan, Zheng page = osd_data->pages[j]; 7405b64640cSYan, Zheng BUG_ON(!page); 7415b64640cSYan, Zheng WARN_ON(!PageUptodate(page)); 7425b64640cSYan, Zheng 7435b64640cSYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 7445b64640cSYan, Zheng CONGESTION_OFF_THRESH( 7455b64640cSYan, Zheng fsc->mount_options->congestion_kb)) 74609dc9fc2SJan Kara clear_bdi_congested(inode_to_bdi(inode), 7475b64640cSYan, Zheng BLK_RW_ASYNC); 7485b64640cSYan, Zheng 7495b64640cSYan, Zheng ceph_put_snap_context(page_snap_context(page)); 7505b64640cSYan, Zheng page->private = 0; 7515b64640cSYan, Zheng ClearPagePrivate(page); 7525b64640cSYan, Zheng dout("unlocking %p\n", page); 7535b64640cSYan, Zheng end_page_writeback(page); 7545b64640cSYan, Zheng 7555b64640cSYan, Zheng if (remove_page) 7565b64640cSYan, Zheng generic_error_remove_page(inode->i_mapping, 7575b64640cSYan, Zheng page); 758e63dc5c7SYehuda Sadeh 7591d3576fdSSage Weil unlock_page(page); 7601d3576fdSSage Weil } 7615b64640cSYan, Zheng dout("writepages_finish %p wrote %llu bytes cleaned %d pages\n", 7625b64640cSYan, Zheng inode, osd_data->length, rc >= 0 ? num_pages : 0); 7631d3576fdSSage Weil 76487060c10SAlex Elder ceph_release_pages(osd_data->pages, num_pages); 7655b64640cSYan, Zheng } 7665b64640cSYan, Zheng 7675b64640cSYan, Zheng ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); 7685b64640cSYan, Zheng 7695b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, 0); 77087060c10SAlex Elder if (osd_data->pages_from_pool) 77187060c10SAlex Elder mempool_free(osd_data->pages, 772640ef79dSCheng Renquan ceph_sb_to_client(inode->i_sb)->wb_pagevec_pool); 7731d3576fdSSage Weil else 77487060c10SAlex Elder kfree(osd_data->pages); 7751d3576fdSSage Weil ceph_osdc_put_request(req); 7761d3576fdSSage Weil } 7771d3576fdSSage Weil 7781d3576fdSSage Weil /* 7791d3576fdSSage Weil * initiate async writeback 7801d3576fdSSage Weil */ 7811d3576fdSSage Weil static int ceph_writepages_start(struct address_space *mapping, 7821d3576fdSSage Weil struct writeback_control *wbc) 7831d3576fdSSage Weil { 7841d3576fdSSage Weil struct inode *inode = mapping->host; 7851d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 786fc2744aaSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 787fc2744aaSYan, Zheng struct ceph_vino vino = ceph_vino(inode); 7882a2d927eSYan, Zheng pgoff_t index, start_index, end = -1; 78980e755feSSage Weil struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; 7901d3576fdSSage Weil struct pagevec pvec; 7911d3576fdSSage Weil int rc = 0; 79293407472SFabian Frederick unsigned int wsize = i_blocksize(inode); 7931d3576fdSSage Weil struct ceph_osd_request *req = NULL; 7941f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 795590e9d98SYan, Zheng bool should_loop, range_whole = false; 796590e9d98SYan, Zheng bool stop, done = false; 7971d3576fdSSage Weil 7983fb99d48SYanhu Cao dout("writepages_start %p (mode=%s)\n", inode, 7991d3576fdSSage Weil wbc->sync_mode == WB_SYNC_NONE ? "NONE" : 8001d3576fdSSage Weil (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); 8011d3576fdSSage Weil 80252953d55SSeraphime Kirkovski if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { 8036c93df5dSYan, Zheng if (ci->i_wrbuffer_ref > 0) { 8046c93df5dSYan, Zheng pr_warn_ratelimited( 8056c93df5dSYan, Zheng "writepage_start %p %lld forced umount\n", 8066c93df5dSYan, Zheng inode, ceph_ino(inode)); 8076c93df5dSYan, Zheng } 808a341d4dfSYan, Zheng mapping_set_error(mapping, -EIO); 8091d3576fdSSage Weil return -EIO; /* we're in a forced umount, don't write! */ 8101d3576fdSSage Weil } 81195cca2b4SYan, Zheng if (fsc->mount_options->wsize < wsize) 8123d14c5d2SYehuda Sadeh wsize = fsc->mount_options->wsize; 8131d3576fdSSage Weil 8141d3576fdSSage Weil pagevec_init(&pvec, 0); 8151d3576fdSSage Weil 816590e9d98SYan, Zheng start_index = wbc->range_cyclic ? mapping->writeback_index : 0; 817590e9d98SYan, Zheng index = start_index; 8181d3576fdSSage Weil 8191d3576fdSSage Weil retry: 8201d3576fdSSage Weil /* find oldest snap context with dirty data */ 82105455e11SYan, Zheng snapc = get_oldest_context(inode, &ceph_wbc, NULL); 8221d3576fdSSage Weil if (!snapc) { 8231d3576fdSSage Weil /* hmm, why does writepages get called when there 8241d3576fdSSage Weil is no dirty data? */ 8251d3576fdSSage Weil dout(" no snap context with dirty data?\n"); 8261d3576fdSSage Weil goto out; 8271d3576fdSSage Weil } 8281d3576fdSSage Weil dout(" oldest snapc is %p seq %lld (%d snaps)\n", 8291d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 830fc2744aaSYan, Zheng 8312a2d927eSYan, Zheng should_loop = false; 8322a2d927eSYan, Zheng if (ceph_wbc.head_snapc && snapc != last_snapc) { 8332a2d927eSYan, Zheng /* where to start/end? */ 8342a2d927eSYan, Zheng if (wbc->range_cyclic) { 8352a2d927eSYan, Zheng index = start_index; 8362a2d927eSYan, Zheng end = -1; 8372a2d927eSYan, Zheng if (index > 0) 8382a2d927eSYan, Zheng should_loop = true; 8392a2d927eSYan, Zheng dout(" cyclic, start at %lu\n", index); 8402a2d927eSYan, Zheng } else { 8412a2d927eSYan, Zheng index = wbc->range_start >> PAGE_SHIFT; 8422a2d927eSYan, Zheng end = wbc->range_end >> PAGE_SHIFT; 8432a2d927eSYan, Zheng if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 8442a2d927eSYan, Zheng range_whole = true; 8452a2d927eSYan, Zheng dout(" not cyclic, %lu to %lu\n", index, end); 8461d3576fdSSage Weil } 8472a2d927eSYan, Zheng } else if (!ceph_wbc.head_snapc) { 8482a2d927eSYan, Zheng /* Do not respect wbc->range_{start,end}. Dirty pages 8492a2d927eSYan, Zheng * in that range can be associated with newer snapc. 8502a2d927eSYan, Zheng * They are not writeable until we write all dirty pages 8512a2d927eSYan, Zheng * associated with 'snapc' get written */ 8522a2d927eSYan, Zheng if (index > 0 || wbc->sync_mode != WB_SYNC_NONE) 8532a2d927eSYan, Zheng should_loop = true; 8542a2d927eSYan, Zheng dout(" non-head snapc, range whole\n"); 8552a2d927eSYan, Zheng } 8562a2d927eSYan, Zheng 8572a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 8581d3576fdSSage Weil last_snapc = snapc; 8591d3576fdSSage Weil 860590e9d98SYan, Zheng stop = false; 861590e9d98SYan, Zheng while (!stop && index <= end) { 8625b64640cSYan, Zheng int num_ops = 0, op_idx; 8630e5ecac7SYan, Zheng unsigned i, pvec_pages, max_pages, locked_pages = 0; 8645b64640cSYan, Zheng struct page **pages = NULL, **data_pages; 865e5975c7cSAlex Elder mempool_t *pool = NULL; /* Becomes non-null if mempool used */ 8661d3576fdSSage Weil struct page *page; 8670e5ecac7SYan, Zheng pgoff_t strip_unit_end = 0; 8685b64640cSYan, Zheng u64 offset = 0, len = 0; 8691d3576fdSSage Weil 8700e5ecac7SYan, Zheng max_pages = wsize >> PAGE_SHIFT; 8711d3576fdSSage Weil 8721d3576fdSSage Weil get_more_pages: 8730e5ecac7SYan, Zheng pvec_pages = min_t(unsigned, PAGEVEC_SIZE, 8740e5ecac7SYan, Zheng max_pages - locked_pages); 8750e5ecac7SYan, Zheng if (end - index < (u64)(pvec_pages - 1)) 8760e5ecac7SYan, Zheng pvec_pages = (unsigned)(end - index) + 1; 8770e5ecac7SYan, Zheng 8781d3576fdSSage Weil pvec_pages = pagevec_lookup_tag(&pvec, mapping, &index, 8791d3576fdSSage Weil PAGECACHE_TAG_DIRTY, 8800e5ecac7SYan, Zheng pvec_pages); 8811d3576fdSSage Weil dout("pagevec_lookup_tag got %d\n", pvec_pages); 8821d3576fdSSage Weil if (!pvec_pages && !locked_pages) 8831d3576fdSSage Weil break; 8841d3576fdSSage Weil for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { 8851d3576fdSSage Weil page = pvec.pages[i]; 8861d3576fdSSage Weil dout("? %p idx %lu\n", page, page->index); 8871d3576fdSSage Weil if (locked_pages == 0) 8881d3576fdSSage Weil lock_page(page); /* first page */ 8891d3576fdSSage Weil else if (!trylock_page(page)) 8901d3576fdSSage Weil break; 8911d3576fdSSage Weil 8921d3576fdSSage Weil /* only dirty pages, or our accounting breaks */ 8931d3576fdSSage Weil if (unlikely(!PageDirty(page)) || 8941d3576fdSSage Weil unlikely(page->mapping != mapping)) { 8951d3576fdSSage Weil dout("!dirty or !mapping %p\n", page); 8961d3576fdSSage Weil unlock_page(page); 8970713e5f2SYan, Zheng continue; 8981d3576fdSSage Weil } 899590e9d98SYan, Zheng if (page->index > end) { 9001d3576fdSSage Weil dout("end of range %p\n", page); 901590e9d98SYan, Zheng /* can't be range_cyclic (1st pass) because 902590e9d98SYan, Zheng * end == -1 in that case. */ 9032a2d927eSYan, Zheng stop = true; 9042a2d927eSYan, Zheng if (ceph_wbc.head_snapc) 9052a2d927eSYan, Zheng done = true; 9061d3576fdSSage Weil unlock_page(page); 9071d3576fdSSage Weil break; 9081d3576fdSSage Weil } 9095b64640cSYan, Zheng if (strip_unit_end && (page->index > strip_unit_end)) { 9105b64640cSYan, Zheng dout("end of strip unit %p\n", page); 9111d3576fdSSage Weil unlock_page(page); 9121d3576fdSSage Weil break; 9131d3576fdSSage Weil } 9141f934b00SYan, Zheng if (page_offset(page) >= ceph_wbc.i_size) { 9151f934b00SYan, Zheng dout("%p page eof %llu\n", 9161f934b00SYan, Zheng page, ceph_wbc.i_size); 917590e9d98SYan, Zheng /* not done if range_cyclic */ 918590e9d98SYan, Zheng stop = true; 9191d3576fdSSage Weil unlock_page(page); 9201d3576fdSSage Weil break; 9211d3576fdSSage Weil } 9221d3576fdSSage Weil if (PageWriteback(page)) { 9230713e5f2SYan, Zheng if (wbc->sync_mode == WB_SYNC_NONE) { 9241d3576fdSSage Weil dout("%p under writeback\n", page); 9251d3576fdSSage Weil unlock_page(page); 9260713e5f2SYan, Zheng continue; 9270713e5f2SYan, Zheng } 9280713e5f2SYan, Zheng dout("waiting on writeback %p\n", page); 9290713e5f2SYan, Zheng wait_on_page_writeback(page); 9301d3576fdSSage Weil } 9311d3576fdSSage Weil 9321d3576fdSSage Weil /* only if matching snap context */ 93361600ef8SYan, Zheng pgsnapc = page_snap_context(page); 9347e1ee54aSYan, Zheng if (pgsnapc != snapc) { 9357e1ee54aSYan, Zheng dout("page snapc %p %lld != oldest %p %lld\n", 93680e755feSSage Weil pgsnapc, pgsnapc->seq, snapc, snapc->seq); 9371d3576fdSSage Weil unlock_page(page); 9380713e5f2SYan, Zheng continue; 9391d3576fdSSage Weil } 9401d3576fdSSage Weil 9411d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) { 9421d3576fdSSage Weil dout("%p !clear_page_dirty_for_io\n", page); 9431d3576fdSSage Weil unlock_page(page); 9440713e5f2SYan, Zheng continue; 9451d3576fdSSage Weil } 9461d3576fdSSage Weil 947e5975c7cSAlex Elder /* 948e5975c7cSAlex Elder * We have something to write. If this is 949e5975c7cSAlex Elder * the first locked page this time through, 9505b64640cSYan, Zheng * calculate max possinle write size and 9515b64640cSYan, Zheng * allocate a page array 952e5975c7cSAlex Elder */ 9531d3576fdSSage Weil if (locked_pages == 0) { 9545b64640cSYan, Zheng u64 objnum; 9555b64640cSYan, Zheng u64 objoff; 9565b64640cSYan, Zheng 9571d3576fdSSage Weil /* prepare async write request */ 9586285bc23SAlex Elder offset = (u64)page_offset(page); 9591d3576fdSSage Weil len = wsize; 9605b64640cSYan, Zheng 9615b64640cSYan, Zheng rc = ceph_calc_file_object_mapping(&ci->i_layout, 9625b64640cSYan, Zheng offset, len, 9635b64640cSYan, Zheng &objnum, &objoff, 9645b64640cSYan, Zheng &len); 9655b64640cSYan, Zheng if (rc < 0) { 9668c71897bSHenry C Chang unlock_page(page); 9678c71897bSHenry C Chang break; 9688c71897bSHenry C Chang } 9698c71897bSHenry C Chang 9703fb99d48SYanhu Cao num_ops = 1; 9715b64640cSYan, Zheng strip_unit_end = page->index + 97209cbfeafSKirill A. Shutemov ((len - 1) >> PAGE_SHIFT); 973715e4cd4SYan, Zheng 9745b64640cSYan, Zheng BUG_ON(pages); 97588486957SAlex Elder max_pages = calc_pages_for(0, (u64)len); 976fc2744aaSYan, Zheng pages = kmalloc(max_pages * sizeof (*pages), 977fc2744aaSYan, Zheng GFP_NOFS); 97888486957SAlex Elder if (!pages) { 97988486957SAlex Elder pool = fsc->wb_pagevec_pool; 98088486957SAlex Elder pages = mempool_alloc(pool, GFP_NOFS); 981e5975c7cSAlex Elder BUG_ON(!pages); 98288486957SAlex Elder } 9835b64640cSYan, Zheng 9845b64640cSYan, Zheng len = 0; 9855b64640cSYan, Zheng } else if (page->index != 98609cbfeafSKirill A. Shutemov (offset + len) >> PAGE_SHIFT) { 9875b64640cSYan, Zheng if (num_ops >= (pool ? CEPH_OSD_SLAB_OPS : 9885b64640cSYan, Zheng CEPH_OSD_MAX_OPS)) { 9895b64640cSYan, Zheng redirty_page_for_writepage(wbc, page); 9905b64640cSYan, Zheng unlock_page(page); 9915b64640cSYan, Zheng break; 9925b64640cSYan, Zheng } 9935b64640cSYan, Zheng 9945b64640cSYan, Zheng num_ops++; 9955b64640cSYan, Zheng offset = (u64)page_offset(page); 9965b64640cSYan, Zheng len = 0; 9971d3576fdSSage Weil } 9981d3576fdSSage Weil 9991d3576fdSSage Weil /* note position of first page in pvec */ 10001d3576fdSSage Weil dout("%p will write page %p idx %lu\n", 10011d3576fdSSage Weil inode, page, page->index); 10022baba250SYehuda Sadeh 10035b64640cSYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 10045b64640cSYan, Zheng CONGESTION_ON_THRESH( 10053d14c5d2SYehuda Sadeh fsc->mount_options->congestion_kb)) { 100609dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), 1007213c99eeSSage Weil BLK_RW_ASYNC); 10082baba250SYehuda Sadeh } 10092baba250SYehuda Sadeh 10100713e5f2SYan, Zheng 10110713e5f2SYan, Zheng pages[locked_pages++] = page; 10120713e5f2SYan, Zheng pvec.pages[i] = NULL; 10130713e5f2SYan, Zheng 101409cbfeafSKirill A. Shutemov len += PAGE_SIZE; 10151d3576fdSSage Weil } 10161d3576fdSSage Weil 10171d3576fdSSage Weil /* did we get anything? */ 10181d3576fdSSage Weil if (!locked_pages) 10191d3576fdSSage Weil goto release_pvec_pages; 10201d3576fdSSage Weil if (i) { 10210713e5f2SYan, Zheng unsigned j, n = 0; 10220713e5f2SYan, Zheng /* shift unused page to beginning of pvec */ 10230713e5f2SYan, Zheng for (j = 0; j < pvec_pages; j++) { 10240713e5f2SYan, Zheng if (!pvec.pages[j]) 10250713e5f2SYan, Zheng continue; 10260713e5f2SYan, Zheng if (n < j) 10270713e5f2SYan, Zheng pvec.pages[n] = pvec.pages[j]; 10280713e5f2SYan, Zheng n++; 10290713e5f2SYan, Zheng } 10300713e5f2SYan, Zheng pvec.nr = n; 10311d3576fdSSage Weil 10321d3576fdSSage Weil if (pvec_pages && i == pvec_pages && 10331d3576fdSSage Weil locked_pages < max_pages) { 10341d3576fdSSage Weil dout("reached end pvec, trying for more\n"); 10350713e5f2SYan, Zheng pagevec_release(&pvec); 10361d3576fdSSage Weil goto get_more_pages; 10371d3576fdSSage Weil } 10381d3576fdSSage Weil } 10391d3576fdSSage Weil 10405b64640cSYan, Zheng new_request: 1041e5975c7cSAlex Elder offset = page_offset(pages[0]); 10425b64640cSYan, Zheng len = wsize; 10435b64640cSYan, Zheng 10445b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 10455b64640cSYan, Zheng &ci->i_layout, vino, 10465b64640cSYan, Zheng offset, &len, 0, num_ops, 10471f934b00SYan, Zheng CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 10481f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 10491f934b00SYan, Zheng ceph_wbc.truncate_size, false); 10505b64640cSYan, Zheng if (IS_ERR(req)) { 10515b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 10525b64640cSYan, Zheng &ci->i_layout, vino, 10535b64640cSYan, Zheng offset, &len, 0, 10545b64640cSYan, Zheng min(num_ops, 10555b64640cSYan, Zheng CEPH_OSD_SLAB_OPS), 10565b64640cSYan, Zheng CEPH_OSD_OP_WRITE, 105754ea0046SIlya Dryomov CEPH_OSD_FLAG_WRITE, 10581f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 10591f934b00SYan, Zheng ceph_wbc.truncate_size, true); 10605b64640cSYan, Zheng BUG_ON(IS_ERR(req)); 10615b64640cSYan, Zheng } 10625b64640cSYan, Zheng BUG_ON(len < page_offset(pages[locked_pages - 1]) + 106309cbfeafSKirill A. Shutemov PAGE_SIZE - offset); 10645b64640cSYan, Zheng 10655b64640cSYan, Zheng req->r_callback = writepages_finish; 10665b64640cSYan, Zheng req->r_inode = inode; 10675b64640cSYan, Zheng 10685b64640cSYan, Zheng /* Format the osd request message and submit the write */ 10695b64640cSYan, Zheng len = 0; 10705b64640cSYan, Zheng data_pages = pages; 10715b64640cSYan, Zheng op_idx = 0; 10725b64640cSYan, Zheng for (i = 0; i < locked_pages; i++) { 10735b64640cSYan, Zheng u64 cur_offset = page_offset(pages[i]); 10745b64640cSYan, Zheng if (offset + len != cur_offset) { 10753fb99d48SYanhu Cao if (op_idx + 1 == req->r_num_ops) 10765b64640cSYan, Zheng break; 10775b64640cSYan, Zheng osd_req_op_extent_dup_last(req, op_idx, 10785b64640cSYan, Zheng cur_offset - offset); 10795b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", 10805b64640cSYan, Zheng offset, len); 10815b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, 10825b64640cSYan, Zheng data_pages, len, 0, 10835b64640cSYan, Zheng !!pool, false); 10845b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 10855b64640cSYan, Zheng 10865b64640cSYan, Zheng len = 0; 10875b64640cSYan, Zheng offset = cur_offset; 10885b64640cSYan, Zheng data_pages = pages + i; 10895b64640cSYan, Zheng op_idx++; 10905b64640cSYan, Zheng } 10915b64640cSYan, Zheng 10925b64640cSYan, Zheng set_page_writeback(pages[i]); 109309cbfeafSKirill A. Shutemov len += PAGE_SIZE; 10945b64640cSYan, Zheng } 10955b64640cSYan, Zheng 10961f934b00SYan, Zheng if (ceph_wbc.size_stable) { 10971f934b00SYan, Zheng len = min(len, ceph_wbc.i_size - offset); 10985b64640cSYan, Zheng } else if (i == locked_pages) { 1099e1966b49SYan, Zheng /* writepages_finish() clears writeback pages 1100e1966b49SYan, Zheng * according to the data length, so make sure 1101e1966b49SYan, Zheng * data length covers all locked pages */ 110209cbfeafSKirill A. Shutemov u64 min_len = len + 1 - PAGE_SIZE; 11031f934b00SYan, Zheng len = get_writepages_data_length(inode, pages[i - 1], 11041f934b00SYan, Zheng offset); 11055b64640cSYan, Zheng len = max(len, min_len); 1106e1966b49SYan, Zheng } 11075b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", offset, len); 11081d3576fdSSage Weil 11095b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, data_pages, len, 11105b64640cSYan, Zheng 0, !!pool, false); 11115b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 1112e5975c7cSAlex Elder 11135b64640cSYan, Zheng BUG_ON(op_idx + 1 != req->r_num_ops); 11145b64640cSYan, Zheng 1115e5975c7cSAlex Elder pool = NULL; 11165b64640cSYan, Zheng if (i < locked_pages) { 11175b64640cSYan, Zheng BUG_ON(num_ops <= req->r_num_ops); 11185b64640cSYan, Zheng num_ops -= req->r_num_ops; 11195b64640cSYan, Zheng locked_pages -= i; 1120e5975c7cSAlex Elder 11215b64640cSYan, Zheng /* allocate new pages array for next request */ 11225b64640cSYan, Zheng data_pages = pages; 11235b64640cSYan, Zheng pages = kmalloc(locked_pages * sizeof (*pages), 11245b64640cSYan, Zheng GFP_NOFS); 11255b64640cSYan, Zheng if (!pages) { 11265b64640cSYan, Zheng pool = fsc->wb_pagevec_pool; 11275b64640cSYan, Zheng pages = mempool_alloc(pool, GFP_NOFS); 11285b64640cSYan, Zheng BUG_ON(!pages); 11295b64640cSYan, Zheng } 11305b64640cSYan, Zheng memcpy(pages, data_pages + i, 11315b64640cSYan, Zheng locked_pages * sizeof(*pages)); 11325b64640cSYan, Zheng memset(data_pages + i, 0, 11335b64640cSYan, Zheng locked_pages * sizeof(*pages)); 11345b64640cSYan, Zheng } else { 11355b64640cSYan, Zheng BUG_ON(num_ops != req->r_num_ops); 11365b64640cSYan, Zheng index = pages[i - 1]->index + 1; 11375b64640cSYan, Zheng /* request message now owns the pages array */ 11385b64640cSYan, Zheng pages = NULL; 11395b64640cSYan, Zheng } 1140e5975c7cSAlex Elder 1141bb873b53SIlya Dryomov req->r_mtime = inode->i_mtime; 11429d6fcb08SSage Weil rc = ceph_osdc_start_request(&fsc->client->osdc, req, true); 11439d6fcb08SSage Weil BUG_ON(rc); 11441d3576fdSSage Weil req = NULL; 11451d3576fdSSage Weil 11465b64640cSYan, Zheng wbc->nr_to_write -= i; 11475b64640cSYan, Zheng if (pages) 11485b64640cSYan, Zheng goto new_request; 11495b64640cSYan, Zheng 11502a2d927eSYan, Zheng /* 11512a2d927eSYan, Zheng * We stop writing back only if we are not doing 11522a2d927eSYan, Zheng * integrity sync. In case of integrity sync we have to 11532a2d927eSYan, Zheng * keep going until we have written all the pages 11542a2d927eSYan, Zheng * we tagged for writeback prior to entering this loop. 11552a2d927eSYan, Zheng */ 11562a2d927eSYan, Zheng if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) 11572a2d927eSYan, Zheng done = stop = true; 11581d3576fdSSage Weil 11591d3576fdSSage Weil release_pvec_pages: 11601d3576fdSSage Weil dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, 11611d3576fdSSage Weil pvec.nr ? pvec.pages[0] : NULL); 11621d3576fdSSage Weil pagevec_release(&pvec); 11631d3576fdSSage Weil } 11641d3576fdSSage Weil 11651d3576fdSSage Weil if (should_loop && !done) { 11661d3576fdSSage Weil /* more to do; loop back to beginning of file */ 11671d3576fdSSage Weil dout("writepages looping back to beginning of file\n"); 11682a2d927eSYan, Zheng end = start_index - 1; /* OK even when start_index == 0 */ 1169f275635eSYan, Zheng 1170f275635eSYan, Zheng /* to write dirty pages associated with next snapc, 1171f275635eSYan, Zheng * we need to wait until current writes complete */ 1172f275635eSYan, Zheng if (wbc->sync_mode != WB_SYNC_NONE && 1173f275635eSYan, Zheng start_index == 0 && /* all dirty pages were checked */ 1174f275635eSYan, Zheng !ceph_wbc.head_snapc) { 1175f275635eSYan, Zheng struct page *page; 1176f275635eSYan, Zheng unsigned i, nr; 1177f275635eSYan, Zheng index = 0; 1178f275635eSYan, Zheng while ((index <= end) && 1179f275635eSYan, Zheng (nr = pagevec_lookup_tag(&pvec, mapping, &index, 1180f275635eSYan, Zheng PAGECACHE_TAG_WRITEBACK, 1181f275635eSYan, Zheng PAGEVEC_SIZE))) { 1182f275635eSYan, Zheng for (i = 0; i < nr; i++) { 1183f275635eSYan, Zheng page = pvec.pages[i]; 1184f275635eSYan, Zheng if (page_snap_context(page) != snapc) 1185f275635eSYan, Zheng continue; 1186f275635eSYan, Zheng wait_on_page_writeback(page); 1187f275635eSYan, Zheng } 1188f275635eSYan, Zheng pagevec_release(&pvec); 1189f275635eSYan, Zheng cond_resched(); 1190f275635eSYan, Zheng } 1191f275635eSYan, Zheng } 1192f275635eSYan, Zheng 11932a2d927eSYan, Zheng start_index = 0; 11941d3576fdSSage Weil index = 0; 11951d3576fdSSage Weil goto retry; 11961d3576fdSSage Weil } 11971d3576fdSSage Weil 11981d3576fdSSage Weil if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 11991d3576fdSSage Weil mapping->writeback_index = index; 12001d3576fdSSage Weil 12011d3576fdSSage Weil out: 12021d3576fdSSage Weil ceph_osdc_put_request(req); 12032a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 12042a2d927eSYan, Zheng dout("writepages dend - startone, rc = %d\n", rc); 12051d3576fdSSage Weil return rc; 12061d3576fdSSage Weil } 12071d3576fdSSage Weil 12081d3576fdSSage Weil 12091d3576fdSSage Weil 12101d3576fdSSage Weil /* 12111d3576fdSSage Weil * See if a given @snapc is either writeable, or already written. 12121d3576fdSSage Weil */ 12131d3576fdSSage Weil static int context_is_writeable_or_written(struct inode *inode, 12141d3576fdSSage Weil struct ceph_snap_context *snapc) 12151d3576fdSSage Weil { 121605455e11SYan, Zheng struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL); 12176298a337SSage Weil int ret = !oldest || snapc->seq <= oldest->seq; 12186298a337SSage Weil 12196298a337SSage Weil ceph_put_snap_context(oldest); 12206298a337SSage Weil return ret; 12211d3576fdSSage Weil } 12221d3576fdSSage Weil 12231d3576fdSSage Weil /* 12241d3576fdSSage Weil * We are only allowed to write into/dirty the page if the page is 12251d3576fdSSage Weil * clean, or already dirty within the same snap context. 12268f883c24SSage Weil * 12278f883c24SSage Weil * called with page locked. 12288f883c24SSage Weil * return success with page locked, 12298f883c24SSage Weil * or any failure (incl -EAGAIN) with page unlocked. 12301d3576fdSSage Weil */ 12314af6b225SYehuda Sadeh static int ceph_update_writeable_page(struct file *file, 12324af6b225SYehuda Sadeh loff_t pos, unsigned len, 12334af6b225SYehuda Sadeh struct page *page) 12341d3576fdSSage Weil { 1235496ad9aaSAl Viro struct inode *inode = file_inode(file); 12366c93df5dSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 12371d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 123809cbfeafSKirill A. Shutemov loff_t page_off = pos & PAGE_MASK; 123909cbfeafSKirill A. Shutemov int pos_in_page = pos & ~PAGE_MASK; 12401d3576fdSSage Weil int end_in_page = pos_in_page + len; 12411d3576fdSSage Weil loff_t i_size; 12421d3576fdSSage Weil int r; 124380e755feSSage Weil struct ceph_snap_context *snapc, *oldest; 12441d3576fdSSage Weil 124552953d55SSeraphime Kirkovski if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { 12466c93df5dSYan, Zheng dout(" page %p forced umount\n", page); 12476c93df5dSYan, Zheng unlock_page(page); 12486c93df5dSYan, Zheng return -EIO; 12496c93df5dSYan, Zheng } 12506c93df5dSYan, Zheng 12511d3576fdSSage Weil retry_locked: 12521d3576fdSSage Weil /* writepages currently holds page lock, but if we change that later, */ 12531d3576fdSSage Weil wait_on_page_writeback(page); 12541d3576fdSSage Weil 125561600ef8SYan, Zheng snapc = page_snap_context(page); 125680e755feSSage Weil if (snapc && snapc != ci->i_head_snapc) { 12571d3576fdSSage Weil /* 12581d3576fdSSage Weil * this page is already dirty in another (older) snap 12591d3576fdSSage Weil * context! is it writeable now? 12601d3576fdSSage Weil */ 126105455e11SYan, Zheng oldest = get_oldest_context(inode, NULL, NULL); 126280e755feSSage Weil if (snapc->seq > oldest->seq) { 12636298a337SSage Weil ceph_put_snap_context(oldest); 12641d3576fdSSage Weil dout(" page %p snapc %p not current or oldest\n", 12656298a337SSage Weil page, snapc); 12661d3576fdSSage Weil /* 12671d3576fdSSage Weil * queue for writeback, and wait for snapc to 12681d3576fdSSage Weil * be writeable or written 12691d3576fdSSage Weil */ 12706298a337SSage Weil snapc = ceph_get_snap_context(snapc); 12711d3576fdSSage Weil unlock_page(page); 12723c6f6b79SSage Weil ceph_queue_writeback(inode); 1273a78bbd4bSYan, Zheng r = wait_event_killable(ci->i_cap_wq, 12741d3576fdSSage Weil context_is_writeable_or_written(inode, snapc)); 12751d3576fdSSage Weil ceph_put_snap_context(snapc); 12768f883c24SSage Weil if (r == -ERESTARTSYS) 12778f883c24SSage Weil return r; 12784af6b225SYehuda Sadeh return -EAGAIN; 12791d3576fdSSage Weil } 12806298a337SSage Weil ceph_put_snap_context(oldest); 12811d3576fdSSage Weil 12821d3576fdSSage Weil /* yay, writeable, do it now (without dropping page lock) */ 12831d3576fdSSage Weil dout(" page %p snapc %p not current, but oldest\n", 12841d3576fdSSage Weil page, snapc); 12851d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) 12861d3576fdSSage Weil goto retry_locked; 12871d3576fdSSage Weil r = writepage_nounlock(page, NULL); 12881d3576fdSSage Weil if (r < 0) 1289dd2bc473SYan, Zheng goto fail_unlock; 12901d3576fdSSage Weil goto retry_locked; 12911d3576fdSSage Weil } 12921d3576fdSSage Weil 12931d3576fdSSage Weil if (PageUptodate(page)) { 12941d3576fdSSage Weil dout(" page %p already uptodate\n", page); 12951d3576fdSSage Weil return 0; 12961d3576fdSSage Weil } 12971d3576fdSSage Weil 12981d3576fdSSage Weil /* full page? */ 129909cbfeafSKirill A. Shutemov if (pos_in_page == 0 && len == PAGE_SIZE) 13001d3576fdSSage Weil return 0; 13011d3576fdSSage Weil 13021d3576fdSSage Weil /* past end of file? */ 130399c88e69SYan, Zheng i_size = i_size_read(inode); 13041d3576fdSSage Weil 13051d3576fdSSage Weil if (page_off >= i_size || 13061d3576fdSSage Weil (pos_in_page == 0 && (pos+len) >= i_size && 130709cbfeafSKirill A. Shutemov end_in_page - pos_in_page != PAGE_SIZE)) { 13081d3576fdSSage Weil dout(" zeroing %p 0 - %d and %d - %d\n", 130909cbfeafSKirill A. Shutemov page, pos_in_page, end_in_page, (int)PAGE_SIZE); 13101d3576fdSSage Weil zero_user_segments(page, 13111d3576fdSSage Weil 0, pos_in_page, 131209cbfeafSKirill A. Shutemov end_in_page, PAGE_SIZE); 13131d3576fdSSage Weil return 0; 13141d3576fdSSage Weil } 13151d3576fdSSage Weil 13161d3576fdSSage Weil /* we need to read it. */ 1317dd2bc473SYan, Zheng r = ceph_do_readpage(file, page); 1318dd2bc473SYan, Zheng if (r < 0) { 1319dd2bc473SYan, Zheng if (r == -EINPROGRESS) 1320dd2bc473SYan, Zheng return -EAGAIN; 1321dd2bc473SYan, Zheng goto fail_unlock; 1322dd2bc473SYan, Zheng } 13231d3576fdSSage Weil goto retry_locked; 1324dd2bc473SYan, Zheng fail_unlock: 13251d3576fdSSage Weil unlock_page(page); 13261d3576fdSSage Weil return r; 13271d3576fdSSage Weil } 13281d3576fdSSage Weil 13291d3576fdSSage Weil /* 13304af6b225SYehuda Sadeh * We are only allowed to write into/dirty the page if the page is 13314af6b225SYehuda Sadeh * clean, or already dirty within the same snap context. 13324af6b225SYehuda Sadeh */ 13334af6b225SYehuda Sadeh static int ceph_write_begin(struct file *file, struct address_space *mapping, 13344af6b225SYehuda Sadeh loff_t pos, unsigned len, unsigned flags, 13354af6b225SYehuda Sadeh struct page **pagep, void **fsdata) 13364af6b225SYehuda Sadeh { 1337496ad9aaSAl Viro struct inode *inode = file_inode(file); 13384af6b225SYehuda Sadeh struct page *page; 133909cbfeafSKirill A. Shutemov pgoff_t index = pos >> PAGE_SHIFT; 13407971bd92SSage Weil int r; 13414af6b225SYehuda Sadeh 13424af6b225SYehuda Sadeh do { 13434af6b225SYehuda Sadeh /* get a page */ 13444af6b225SYehuda Sadeh page = grab_cache_page_write_begin(mapping, index, 0); 13457971bd92SSage Weil if (!page) 13467971bd92SSage Weil return -ENOMEM; 13474af6b225SYehuda Sadeh 13484af6b225SYehuda Sadeh dout("write_begin file %p inode %p page %p %d~%d\n", file, 13494af6b225SYehuda Sadeh inode, page, (int)pos, (int)len); 13504af6b225SYehuda Sadeh 13514af6b225SYehuda Sadeh r = ceph_update_writeable_page(file, pos, len, page); 1352c1d00b2dSTaesoo Kim if (r < 0) 135309cbfeafSKirill A. Shutemov put_page(page); 1354c1d00b2dSTaesoo Kim else 1355c1d00b2dSTaesoo Kim *pagep = page; 13564af6b225SYehuda Sadeh } while (r == -EAGAIN); 13574af6b225SYehuda Sadeh 13584af6b225SYehuda Sadeh return r; 13594af6b225SYehuda Sadeh } 13604af6b225SYehuda Sadeh 13614af6b225SYehuda Sadeh /* 13621d3576fdSSage Weil * we don't do anything in here that simple_write_end doesn't do 13635dda377cSYan, Zheng * except adjust dirty page accounting 13641d3576fdSSage Weil */ 13651d3576fdSSage Weil static int ceph_write_end(struct file *file, struct address_space *mapping, 13661d3576fdSSage Weil loff_t pos, unsigned len, unsigned copied, 13671d3576fdSSage Weil struct page *page, void *fsdata) 13681d3576fdSSage Weil { 1369496ad9aaSAl Viro struct inode *inode = file_inode(file); 1370efb0ca76SYan, Zheng bool check_cap = false; 13711d3576fdSSage Weil 13721d3576fdSSage Weil dout("write_end file %p inode %p page %p %d~%d (%d)\n", file, 13731d3576fdSSage Weil inode, page, (int)pos, (int)copied, (int)len); 13741d3576fdSSage Weil 13751d3576fdSSage Weil /* zero the stale part of the page if we did a short copy */ 1376b9de313cSAl Viro if (!PageUptodate(page)) { 1377b9de313cSAl Viro if (copied < len) { 1378b9de313cSAl Viro copied = 0; 1379b9de313cSAl Viro goto out; 1380b9de313cSAl Viro } 1381b9de313cSAl Viro SetPageUptodate(page); 1382b9de313cSAl Viro } 13831d3576fdSSage Weil 13841d3576fdSSage Weil /* did file size increase? */ 138599c88e69SYan, Zheng if (pos+copied > i_size_read(inode)) 13861d3576fdSSage Weil check_cap = ceph_inode_set_size(inode, pos+copied); 13871d3576fdSSage Weil 13881d3576fdSSage Weil set_page_dirty(page); 13891d3576fdSSage Weil 1390b9de313cSAl Viro out: 13911d3576fdSSage Weil unlock_page(page); 139209cbfeafSKirill A. Shutemov put_page(page); 13931d3576fdSSage Weil 13941d3576fdSSage Weil if (check_cap) 13951d3576fdSSage Weil ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY, NULL); 13961d3576fdSSage Weil 13971d3576fdSSage Weil return copied; 13981d3576fdSSage Weil } 13991d3576fdSSage Weil 14001d3576fdSSage Weil /* 14011d3576fdSSage Weil * we set .direct_IO to indicate direct io is supported, but since we 14021d3576fdSSage Weil * intercept O_DIRECT reads and writes early, this function should 14031d3576fdSSage Weil * never get called. 14041d3576fdSSage Weil */ 1405c8b8e32dSChristoph Hellwig static ssize_t ceph_direct_io(struct kiocb *iocb, struct iov_iter *iter) 14061d3576fdSSage Weil { 14071d3576fdSSage Weil WARN_ON(1); 14081d3576fdSSage Weil return -EINVAL; 14091d3576fdSSage Weil } 14101d3576fdSSage Weil 14111d3576fdSSage Weil const struct address_space_operations ceph_aops = { 14121d3576fdSSage Weil .readpage = ceph_readpage, 14131d3576fdSSage Weil .readpages = ceph_readpages, 14141d3576fdSSage Weil .writepage = ceph_writepage, 14151d3576fdSSage Weil .writepages = ceph_writepages_start, 14161d3576fdSSage Weil .write_begin = ceph_write_begin, 14171d3576fdSSage Weil .write_end = ceph_write_end, 14181d3576fdSSage Weil .set_page_dirty = ceph_set_page_dirty, 14191d3576fdSSage Weil .invalidatepage = ceph_invalidatepage, 14201d3576fdSSage Weil .releasepage = ceph_releasepage, 14211d3576fdSSage Weil .direct_IO = ceph_direct_io, 14221d3576fdSSage Weil }; 14231d3576fdSSage Weil 14244f7e89f6SYan, Zheng static void ceph_block_sigs(sigset_t *oldset) 14254f7e89f6SYan, Zheng { 14264f7e89f6SYan, Zheng sigset_t mask; 14274f7e89f6SYan, Zheng siginitsetinv(&mask, sigmask(SIGKILL)); 14284f7e89f6SYan, Zheng sigprocmask(SIG_BLOCK, &mask, oldset); 14294f7e89f6SYan, Zheng } 14304f7e89f6SYan, Zheng 14314f7e89f6SYan, Zheng static void ceph_restore_sigs(sigset_t *oldset) 14324f7e89f6SYan, Zheng { 14334f7e89f6SYan, Zheng sigprocmask(SIG_SETMASK, oldset, NULL); 14344f7e89f6SYan, Zheng } 14351d3576fdSSage Weil 14361d3576fdSSage Weil /* 14371d3576fdSSage Weil * vm ops 14381d3576fdSSage Weil */ 143911bac800SDave Jiang static int ceph_filemap_fault(struct vm_fault *vmf) 144061f68816SYan, Zheng { 144111bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 144261f68816SYan, Zheng struct inode *inode = file_inode(vma->vm_file); 144361f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 144461f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 14453738daa6SYan, Zheng struct page *pinned_page = NULL; 144609cbfeafSKirill A. Shutemov loff_t off = vmf->pgoff << PAGE_SHIFT; 144761f68816SYan, Zheng int want, got, ret; 14484f7e89f6SYan, Zheng sigset_t oldset; 14494f7e89f6SYan, Zheng 14504f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 145161f68816SYan, Zheng 145261f68816SYan, Zheng dout("filemap_fault %p %llx.%llx %llu~%zd trying to get caps\n", 145309cbfeafSKirill A. Shutemov inode, ceph_vinop(inode), off, (size_t)PAGE_SIZE); 145461f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 145561f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; 145661f68816SYan, Zheng else 145761f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE; 14584f7e89f6SYan, Zheng 145961f68816SYan, Zheng got = 0; 14604f7e89f6SYan, Zheng ret = ceph_get_caps(ci, CEPH_CAP_FILE_RD, want, -1, &got, &pinned_page); 14616ce026e4SYan, Zheng if (ret < 0) 14624f7e89f6SYan, Zheng goto out_restore; 14636ce026e4SYan, Zheng 146461f68816SYan, Zheng dout("filemap_fault %p %llu~%zd got cap refs on %s\n", 146509cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ceph_cap_string(got)); 146661f68816SYan, Zheng 146783701246SYan, Zheng if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || 14682b1ac852SYan, Zheng ci->i_inline_version == CEPH_INLINE_NONE) { 14692b1ac852SYan, Zheng current->journal_info = vma->vm_file; 147011bac800SDave Jiang ret = filemap_fault(vmf); 14712b1ac852SYan, Zheng current->journal_info = NULL; 14722b1ac852SYan, Zheng } else 147383701246SYan, Zheng ret = -EAGAIN; 147461f68816SYan, Zheng 147561f68816SYan, Zheng dout("filemap_fault %p %llu~%zd dropping cap refs on %s ret %d\n", 147609cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ceph_cap_string(got), ret); 14773738daa6SYan, Zheng if (pinned_page) 147809cbfeafSKirill A. Shutemov put_page(pinned_page); 147961f68816SYan, Zheng ceph_put_cap_refs(ci, got); 148061f68816SYan, Zheng 148183701246SYan, Zheng if (ret != -EAGAIN) 14824f7e89f6SYan, Zheng goto out_restore; 148383701246SYan, Zheng 148483701246SYan, Zheng /* read inline data */ 148509cbfeafSKirill A. Shutemov if (off >= PAGE_SIZE) { 148683701246SYan, Zheng /* does not support inline data > PAGE_SIZE */ 148783701246SYan, Zheng ret = VM_FAULT_SIGBUS; 148883701246SYan, Zheng } else { 148983701246SYan, Zheng int ret1; 149083701246SYan, Zheng struct address_space *mapping = inode->i_mapping; 149183701246SYan, Zheng struct page *page = find_or_create_page(mapping, 0, 1492c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1493c62d2555SMichal Hocko ~__GFP_FS)); 149483701246SYan, Zheng if (!page) { 149583701246SYan, Zheng ret = VM_FAULT_OOM; 14964f7e89f6SYan, Zheng goto out_inline; 149783701246SYan, Zheng } 149883701246SYan, Zheng ret1 = __ceph_do_getattr(inode, page, 149983701246SYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 150083701246SYan, Zheng if (ret1 < 0 || off >= i_size_read(inode)) { 150183701246SYan, Zheng unlock_page(page); 150209cbfeafSKirill A. Shutemov put_page(page); 15036ce026e4SYan, Zheng if (ret1 < 0) 15046ce026e4SYan, Zheng ret = ret1; 15056ce026e4SYan, Zheng else 150683701246SYan, Zheng ret = VM_FAULT_SIGBUS; 15074f7e89f6SYan, Zheng goto out_inline; 150883701246SYan, Zheng } 150909cbfeafSKirill A. Shutemov if (ret1 < PAGE_SIZE) 151009cbfeafSKirill A. Shutemov zero_user_segment(page, ret1, PAGE_SIZE); 151183701246SYan, Zheng else 151283701246SYan, Zheng flush_dcache_page(page); 151383701246SYan, Zheng SetPageUptodate(page); 151483701246SYan, Zheng vmf->page = page; 151583701246SYan, Zheng ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; 15164f7e89f6SYan, Zheng out_inline: 151783701246SYan, Zheng dout("filemap_fault %p %llu~%zd read inline data ret %d\n", 151809cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ret); 15194f7e89f6SYan, Zheng } 15204f7e89f6SYan, Zheng out_restore: 15214f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 15226ce026e4SYan, Zheng if (ret < 0) 15236ce026e4SYan, Zheng ret = (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 15246ce026e4SYan, Zheng 152561f68816SYan, Zheng return ret; 152661f68816SYan, Zheng } 15271d3576fdSSage Weil 15281d3576fdSSage Weil /* 15291d3576fdSSage Weil * Reuse write_begin here for simplicity. 15301d3576fdSSage Weil */ 153111bac800SDave Jiang static int ceph_page_mkwrite(struct vm_fault *vmf) 15321d3576fdSSage Weil { 153311bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 1534496ad9aaSAl Viro struct inode *inode = file_inode(vma->vm_file); 153561f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 153661f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 1537f66fd9f0SYan, Zheng struct ceph_cap_flush *prealloc_cf; 153861f68816SYan, Zheng struct page *page = vmf->page; 15396285bc23SAlex Elder loff_t off = page_offset(page); 154061f68816SYan, Zheng loff_t size = i_size_read(inode); 154161f68816SYan, Zheng size_t len; 154261f68816SYan, Zheng int want, got, ret; 15434f7e89f6SYan, Zheng sigset_t oldset; 15441d3576fdSSage Weil 1545f66fd9f0SYan, Zheng prealloc_cf = ceph_alloc_cap_flush(); 1546f66fd9f0SYan, Zheng if (!prealloc_cf) 15476ce026e4SYan, Zheng return VM_FAULT_OOM; 1548f66fd9f0SYan, Zheng 15494f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 15501d3576fdSSage Weil 155128127bddSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 155228127bddSYan, Zheng struct page *locked_page = NULL; 155328127bddSYan, Zheng if (off == 0) { 155428127bddSYan, Zheng lock_page(page); 155528127bddSYan, Zheng locked_page = page; 155628127bddSYan, Zheng } 155728127bddSYan, Zheng ret = ceph_uninline_data(vma->vm_file, locked_page); 155828127bddSYan, Zheng if (locked_page) 155928127bddSYan, Zheng unlock_page(locked_page); 15606ce026e4SYan, Zheng if (ret < 0) 1561f66fd9f0SYan, Zheng goto out_free; 1562f66fd9f0SYan, Zheng } 156328127bddSYan, Zheng 156409cbfeafSKirill A. Shutemov if (off + PAGE_SIZE <= size) 156509cbfeafSKirill A. Shutemov len = PAGE_SIZE; 15661d3576fdSSage Weil else 156709cbfeafSKirill A. Shutemov len = size & ~PAGE_MASK; 15681d3576fdSSage Weil 156961f68816SYan, Zheng dout("page_mkwrite %p %llx.%llx %llu~%zd getting caps i_size %llu\n", 157061f68816SYan, Zheng inode, ceph_vinop(inode), off, len, size); 157161f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 157261f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; 157361f68816SYan, Zheng else 157461f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER; 15754f7e89f6SYan, Zheng 157661f68816SYan, Zheng got = 0; 15773738daa6SYan, Zheng ret = ceph_get_caps(ci, CEPH_CAP_FILE_WR, want, off + len, 15783738daa6SYan, Zheng &got, NULL); 15796ce026e4SYan, Zheng if (ret < 0) 1580f66fd9f0SYan, Zheng goto out_free; 15816ce026e4SYan, Zheng 158261f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd got cap refs on %s\n", 158361f68816SYan, Zheng inode, off, len, ceph_cap_string(got)); 158461f68816SYan, Zheng 158561f68816SYan, Zheng /* Update time before taking page lock */ 158661f68816SYan, Zheng file_update_time(vma->vm_file); 15874af6b225SYehuda Sadeh 1588f0b33df5SYan, Zheng do { 15894af6b225SYehuda Sadeh lock_page(page); 15904af6b225SYehuda Sadeh 15916ce026e4SYan, Zheng if ((off > size) || (page->mapping != inode->i_mapping)) { 1592f9cac5acSYan, Zheng unlock_page(page); 15936ce026e4SYan, Zheng ret = VM_FAULT_NOPAGE; 1594f0b33df5SYan, Zheng break; 1595f9cac5acSYan, Zheng } 15964af6b225SYehuda Sadeh 15974af6b225SYehuda Sadeh ret = ceph_update_writeable_page(vma->vm_file, off, len, page); 1598f9cac5acSYan, Zheng if (ret >= 0) { 15994af6b225SYehuda Sadeh /* success. we'll keep the page locked. */ 16001d3576fdSSage Weil set_page_dirty(page); 16011d3576fdSSage Weil ret = VM_FAULT_LOCKED; 16021d3576fdSSage Weil } 1603f0b33df5SYan, Zheng } while (ret == -EAGAIN); 1604f0b33df5SYan, Zheng 160528127bddSYan, Zheng if (ret == VM_FAULT_LOCKED || 160628127bddSYan, Zheng ci->i_inline_version != CEPH_INLINE_NONE) { 160761f68816SYan, Zheng int dirty; 160861f68816SYan, Zheng spin_lock(&ci->i_ceph_lock); 160928127bddSYan, Zheng ci->i_inline_version = CEPH_INLINE_NONE; 1610f66fd9f0SYan, Zheng dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, 1611f66fd9f0SYan, Zheng &prealloc_cf); 161261f68816SYan, Zheng spin_unlock(&ci->i_ceph_lock); 161361f68816SYan, Zheng if (dirty) 161461f68816SYan, Zheng __mark_inode_dirty(inode, dirty); 161561f68816SYan, Zheng } 161661f68816SYan, Zheng 161761f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd dropping cap refs on %s ret %d\n", 161861f68816SYan, Zheng inode, off, len, ceph_cap_string(got), ret); 161961f68816SYan, Zheng ceph_put_cap_refs(ci, got); 1620f66fd9f0SYan, Zheng out_free: 16214f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 1622f66fd9f0SYan, Zheng ceph_free_cap_flush(prealloc_cf); 16236ce026e4SYan, Zheng if (ret < 0) 16246ce026e4SYan, Zheng ret = (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 16251d3576fdSSage Weil return ret; 16261d3576fdSSage Weil } 16271d3576fdSSage Weil 162831c542a1SYan, Zheng void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, 162931c542a1SYan, Zheng char *data, size_t len) 163031c542a1SYan, Zheng { 163131c542a1SYan, Zheng struct address_space *mapping = inode->i_mapping; 163231c542a1SYan, Zheng struct page *page; 163331c542a1SYan, Zheng 163431c542a1SYan, Zheng if (locked_page) { 163531c542a1SYan, Zheng page = locked_page; 163631c542a1SYan, Zheng } else { 163731c542a1SYan, Zheng if (i_size_read(inode) == 0) 163831c542a1SYan, Zheng return; 163931c542a1SYan, Zheng page = find_or_create_page(mapping, 0, 1640c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1641c62d2555SMichal Hocko ~__GFP_FS)); 164231c542a1SYan, Zheng if (!page) 164331c542a1SYan, Zheng return; 164431c542a1SYan, Zheng if (PageUptodate(page)) { 164531c542a1SYan, Zheng unlock_page(page); 164609cbfeafSKirill A. Shutemov put_page(page); 164731c542a1SYan, Zheng return; 164831c542a1SYan, Zheng } 164931c542a1SYan, Zheng } 165031c542a1SYan, Zheng 16510668ff52SIlya Dryomov dout("fill_inline_data %p %llx.%llx len %zu locked_page %p\n", 165231c542a1SYan, Zheng inode, ceph_vinop(inode), len, locked_page); 165331c542a1SYan, Zheng 165431c542a1SYan, Zheng if (len > 0) { 165531c542a1SYan, Zheng void *kaddr = kmap_atomic(page); 165631c542a1SYan, Zheng memcpy(kaddr, data, len); 165731c542a1SYan, Zheng kunmap_atomic(kaddr); 165831c542a1SYan, Zheng } 165931c542a1SYan, Zheng 166031c542a1SYan, Zheng if (page != locked_page) { 166109cbfeafSKirill A. Shutemov if (len < PAGE_SIZE) 166209cbfeafSKirill A. Shutemov zero_user_segment(page, len, PAGE_SIZE); 166331c542a1SYan, Zheng else 166431c542a1SYan, Zheng flush_dcache_page(page); 166531c542a1SYan, Zheng 166631c542a1SYan, Zheng SetPageUptodate(page); 166731c542a1SYan, Zheng unlock_page(page); 166809cbfeafSKirill A. Shutemov put_page(page); 166931c542a1SYan, Zheng } 167031c542a1SYan, Zheng } 167131c542a1SYan, Zheng 167228127bddSYan, Zheng int ceph_uninline_data(struct file *filp, struct page *locked_page) 167328127bddSYan, Zheng { 167428127bddSYan, Zheng struct inode *inode = file_inode(filp); 167528127bddSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 167628127bddSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 167728127bddSYan, Zheng struct ceph_osd_request *req; 167828127bddSYan, Zheng struct page *page = NULL; 167928127bddSYan, Zheng u64 len, inline_version; 168028127bddSYan, Zheng int err = 0; 168128127bddSYan, Zheng bool from_pagecache = false; 168228127bddSYan, Zheng 168328127bddSYan, Zheng spin_lock(&ci->i_ceph_lock); 168428127bddSYan, Zheng inline_version = ci->i_inline_version; 168528127bddSYan, Zheng spin_unlock(&ci->i_ceph_lock); 168628127bddSYan, Zheng 168728127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu\n", 168828127bddSYan, Zheng inode, ceph_vinop(inode), inline_version); 168928127bddSYan, Zheng 169028127bddSYan, Zheng if (inline_version == 1 || /* initial version, no data */ 169128127bddSYan, Zheng inline_version == CEPH_INLINE_NONE) 169228127bddSYan, Zheng goto out; 169328127bddSYan, Zheng 169428127bddSYan, Zheng if (locked_page) { 169528127bddSYan, Zheng page = locked_page; 169628127bddSYan, Zheng WARN_ON(!PageUptodate(page)); 169728127bddSYan, Zheng } else if (ceph_caps_issued(ci) & 169828127bddSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) { 169928127bddSYan, Zheng page = find_get_page(inode->i_mapping, 0); 170028127bddSYan, Zheng if (page) { 170128127bddSYan, Zheng if (PageUptodate(page)) { 170228127bddSYan, Zheng from_pagecache = true; 170328127bddSYan, Zheng lock_page(page); 170428127bddSYan, Zheng } else { 170509cbfeafSKirill A. Shutemov put_page(page); 170628127bddSYan, Zheng page = NULL; 170728127bddSYan, Zheng } 170828127bddSYan, Zheng } 170928127bddSYan, Zheng } 171028127bddSYan, Zheng 171128127bddSYan, Zheng if (page) { 171228127bddSYan, Zheng len = i_size_read(inode); 171309cbfeafSKirill A. Shutemov if (len > PAGE_SIZE) 171409cbfeafSKirill A. Shutemov len = PAGE_SIZE; 171528127bddSYan, Zheng } else { 171628127bddSYan, Zheng page = __page_cache_alloc(GFP_NOFS); 171728127bddSYan, Zheng if (!page) { 171828127bddSYan, Zheng err = -ENOMEM; 171928127bddSYan, Zheng goto out; 172028127bddSYan, Zheng } 172128127bddSYan, Zheng err = __ceph_do_getattr(inode, page, 172228127bddSYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 172328127bddSYan, Zheng if (err < 0) { 172428127bddSYan, Zheng /* no inline data */ 172528127bddSYan, Zheng if (err == -ENODATA) 172628127bddSYan, Zheng err = 0; 172728127bddSYan, Zheng goto out; 172828127bddSYan, Zheng } 172928127bddSYan, Zheng len = err; 173028127bddSYan, Zheng } 173128127bddSYan, Zheng 173228127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 173328127bddSYan, Zheng ceph_vino(inode), 0, &len, 0, 1, 173454ea0046SIlya Dryomov CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, 173534b759b4SIlya Dryomov NULL, 0, 0, false); 173628127bddSYan, Zheng if (IS_ERR(req)) { 173728127bddSYan, Zheng err = PTR_ERR(req); 173828127bddSYan, Zheng goto out; 173928127bddSYan, Zheng } 174028127bddSYan, Zheng 1741bb873b53SIlya Dryomov req->r_mtime = inode->i_mtime; 174228127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 174328127bddSYan, Zheng if (!err) 174428127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 174528127bddSYan, Zheng ceph_osdc_put_request(req); 174628127bddSYan, Zheng if (err < 0) 174728127bddSYan, Zheng goto out; 174828127bddSYan, Zheng 174928127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 175028127bddSYan, Zheng ceph_vino(inode), 0, &len, 1, 3, 175154ea0046SIlya Dryomov CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 175234b759b4SIlya Dryomov NULL, ci->i_truncate_seq, 175334b759b4SIlya Dryomov ci->i_truncate_size, false); 175428127bddSYan, Zheng if (IS_ERR(req)) { 175528127bddSYan, Zheng err = PTR_ERR(req); 175628127bddSYan, Zheng goto out; 175728127bddSYan, Zheng } 175828127bddSYan, Zheng 175928127bddSYan, Zheng osd_req_op_extent_osd_data_pages(req, 1, &page, len, 0, false, false); 176028127bddSYan, Zheng 1761ec137c10SYan, Zheng { 1762ec137c10SYan, Zheng __le64 xattr_buf = cpu_to_le64(inline_version); 176328127bddSYan, Zheng err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, 1764ec137c10SYan, Zheng "inline_version", &xattr_buf, 1765ec137c10SYan, Zheng sizeof(xattr_buf), 176628127bddSYan, Zheng CEPH_OSD_CMPXATTR_OP_GT, 176728127bddSYan, Zheng CEPH_OSD_CMPXATTR_MODE_U64); 176828127bddSYan, Zheng if (err) 176928127bddSYan, Zheng goto out_put; 1770ec137c10SYan, Zheng } 177128127bddSYan, Zheng 1772ec137c10SYan, Zheng { 1773ec137c10SYan, Zheng char xattr_buf[32]; 1774ec137c10SYan, Zheng int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), 1775ec137c10SYan, Zheng "%llu", inline_version); 177628127bddSYan, Zheng err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, 1777ec137c10SYan, Zheng "inline_version", 1778ec137c10SYan, Zheng xattr_buf, xattr_len, 0, 0); 177928127bddSYan, Zheng if (err) 178028127bddSYan, Zheng goto out_put; 1781ec137c10SYan, Zheng } 178228127bddSYan, Zheng 1783bb873b53SIlya Dryomov req->r_mtime = inode->i_mtime; 178428127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 178528127bddSYan, Zheng if (!err) 178628127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 178728127bddSYan, Zheng out_put: 178828127bddSYan, Zheng ceph_osdc_put_request(req); 178928127bddSYan, Zheng if (err == -ECANCELED) 179028127bddSYan, Zheng err = 0; 179128127bddSYan, Zheng out: 179228127bddSYan, Zheng if (page && page != locked_page) { 179328127bddSYan, Zheng if (from_pagecache) { 179428127bddSYan, Zheng unlock_page(page); 179509cbfeafSKirill A. Shutemov put_page(page); 179628127bddSYan, Zheng } else 179728127bddSYan, Zheng __free_pages(page, 0); 179828127bddSYan, Zheng } 179928127bddSYan, Zheng 180028127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu = %d\n", 180128127bddSYan, Zheng inode, ceph_vinop(inode), inline_version, err); 180228127bddSYan, Zheng return err; 180328127bddSYan, Zheng } 180428127bddSYan, Zheng 18057cbea8dcSKirill A. Shutemov static const struct vm_operations_struct ceph_vmops = { 180661f68816SYan, Zheng .fault = ceph_filemap_fault, 18071d3576fdSSage Weil .page_mkwrite = ceph_page_mkwrite, 18081d3576fdSSage Weil }; 18091d3576fdSSage Weil 18101d3576fdSSage Weil int ceph_mmap(struct file *file, struct vm_area_struct *vma) 18111d3576fdSSage Weil { 18121d3576fdSSage Weil struct address_space *mapping = file->f_mapping; 18131d3576fdSSage Weil 18141d3576fdSSage Weil if (!mapping->a_ops->readpage) 18151d3576fdSSage Weil return -ENOEXEC; 18161d3576fdSSage Weil file_accessed(file); 18171d3576fdSSage Weil vma->vm_ops = &ceph_vmops; 18181d3576fdSSage Weil return 0; 18191d3576fdSSage Weil } 182010183a69SYan, Zheng 182110183a69SYan, Zheng enum { 182210183a69SYan, Zheng POOL_READ = 1, 182310183a69SYan, Zheng POOL_WRITE = 2, 182410183a69SYan, Zheng }; 182510183a69SYan, Zheng 1826779fe0fbSYan, Zheng static int __ceph_pool_perm_get(struct ceph_inode_info *ci, 1827779fe0fbSYan, Zheng s64 pool, struct ceph_string *pool_ns) 182810183a69SYan, Zheng { 182910183a69SYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(&ci->vfs_inode); 183010183a69SYan, Zheng struct ceph_mds_client *mdsc = fsc->mdsc; 183110183a69SYan, Zheng struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; 183210183a69SYan, Zheng struct rb_node **p, *parent; 183310183a69SYan, Zheng struct ceph_pool_perm *perm; 183410183a69SYan, Zheng struct page **pages; 1835779fe0fbSYan, Zheng size_t pool_ns_len; 183610183a69SYan, Zheng int err = 0, err2 = 0, have = 0; 183710183a69SYan, Zheng 183810183a69SYan, Zheng down_read(&mdsc->pool_perm_rwsem); 183910183a69SYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 184010183a69SYan, Zheng while (*p) { 184110183a69SYan, Zheng perm = rb_entry(*p, struct ceph_pool_perm, node); 184210183a69SYan, Zheng if (pool < perm->pool) 184310183a69SYan, Zheng p = &(*p)->rb_left; 184410183a69SYan, Zheng else if (pool > perm->pool) 184510183a69SYan, Zheng p = &(*p)->rb_right; 184610183a69SYan, Zheng else { 1847779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1848779fe0fbSYan, Zheng perm->pool_ns, 1849779fe0fbSYan, Zheng perm->pool_ns_len); 1850779fe0fbSYan, Zheng if (ret < 0) 1851779fe0fbSYan, Zheng p = &(*p)->rb_left; 1852779fe0fbSYan, Zheng else if (ret > 0) 1853779fe0fbSYan, Zheng p = &(*p)->rb_right; 1854779fe0fbSYan, Zheng else { 185510183a69SYan, Zheng have = perm->perm; 185610183a69SYan, Zheng break; 185710183a69SYan, Zheng } 185810183a69SYan, Zheng } 1859779fe0fbSYan, Zheng } 186010183a69SYan, Zheng up_read(&mdsc->pool_perm_rwsem); 186110183a69SYan, Zheng if (*p) 186210183a69SYan, Zheng goto out; 186310183a69SYan, Zheng 1864779fe0fbSYan, Zheng if (pool_ns) 1865779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s no perm cached\n", 1866779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str); 1867779fe0fbSYan, Zheng else 18687627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld no perm cached\n", pool); 186910183a69SYan, Zheng 187010183a69SYan, Zheng down_write(&mdsc->pool_perm_rwsem); 1871779fe0fbSYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 187210183a69SYan, Zheng parent = NULL; 187310183a69SYan, Zheng while (*p) { 187410183a69SYan, Zheng parent = *p; 187510183a69SYan, Zheng perm = rb_entry(parent, struct ceph_pool_perm, node); 187610183a69SYan, Zheng if (pool < perm->pool) 187710183a69SYan, Zheng p = &(*p)->rb_left; 187810183a69SYan, Zheng else if (pool > perm->pool) 187910183a69SYan, Zheng p = &(*p)->rb_right; 188010183a69SYan, Zheng else { 1881779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1882779fe0fbSYan, Zheng perm->pool_ns, 1883779fe0fbSYan, Zheng perm->pool_ns_len); 1884779fe0fbSYan, Zheng if (ret < 0) 1885779fe0fbSYan, Zheng p = &(*p)->rb_left; 1886779fe0fbSYan, Zheng else if (ret > 0) 1887779fe0fbSYan, Zheng p = &(*p)->rb_right; 1888779fe0fbSYan, Zheng else { 188910183a69SYan, Zheng have = perm->perm; 189010183a69SYan, Zheng break; 189110183a69SYan, Zheng } 189210183a69SYan, Zheng } 1893779fe0fbSYan, Zheng } 189410183a69SYan, Zheng if (*p) { 189510183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 189610183a69SYan, Zheng goto out; 189710183a69SYan, Zheng } 189810183a69SYan, Zheng 189934b759b4SIlya Dryomov rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 190010183a69SYan, Zheng 1, false, GFP_NOFS); 190110183a69SYan, Zheng if (!rd_req) { 190210183a69SYan, Zheng err = -ENOMEM; 190310183a69SYan, Zheng goto out_unlock; 190410183a69SYan, Zheng } 190510183a69SYan, Zheng 190610183a69SYan, Zheng rd_req->r_flags = CEPH_OSD_FLAG_READ; 190710183a69SYan, Zheng osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); 190810183a69SYan, Zheng rd_req->r_base_oloc.pool = pool; 1909779fe0fbSYan, Zheng if (pool_ns) 1910779fe0fbSYan, Zheng rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns); 1911d30291b9SIlya Dryomov ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ci->i_vino.ino); 191210183a69SYan, Zheng 191313d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(rd_req, GFP_NOFS); 191413d1ad16SIlya Dryomov if (err) 191513d1ad16SIlya Dryomov goto out_unlock; 191610183a69SYan, Zheng 191734b759b4SIlya Dryomov wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 191810183a69SYan, Zheng 1, false, GFP_NOFS); 191910183a69SYan, Zheng if (!wr_req) { 192010183a69SYan, Zheng err = -ENOMEM; 192110183a69SYan, Zheng goto out_unlock; 192210183a69SYan, Zheng } 192310183a69SYan, Zheng 192454ea0046SIlya Dryomov wr_req->r_flags = CEPH_OSD_FLAG_WRITE; 192510183a69SYan, Zheng osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); 192663244fa1SIlya Dryomov ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc); 1927d30291b9SIlya Dryomov ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid); 192810183a69SYan, Zheng 192913d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(wr_req, GFP_NOFS); 193013d1ad16SIlya Dryomov if (err) 193113d1ad16SIlya Dryomov goto out_unlock; 193210183a69SYan, Zheng 193310183a69SYan, Zheng /* one page should be large enough for STAT data */ 193410183a69SYan, Zheng pages = ceph_alloc_page_vector(1, GFP_KERNEL); 193510183a69SYan, Zheng if (IS_ERR(pages)) { 193610183a69SYan, Zheng err = PTR_ERR(pages); 193710183a69SYan, Zheng goto out_unlock; 193810183a69SYan, Zheng } 193910183a69SYan, Zheng 194010183a69SYan, Zheng osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, 194110183a69SYan, Zheng 0, false, true); 194210183a69SYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false); 194310183a69SYan, Zheng 1944bb873b53SIlya Dryomov wr_req->r_mtime = ci->vfs_inode.i_mtime; 1945a1f4020aSJeff Layton wr_req->r_abort_on_full = true; 194610183a69SYan, Zheng err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false); 194710183a69SYan, Zheng 194810183a69SYan, Zheng if (!err) 194910183a69SYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); 195010183a69SYan, Zheng if (!err2) 195110183a69SYan, Zheng err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); 195210183a69SYan, Zheng 195310183a69SYan, Zheng if (err >= 0 || err == -ENOENT) 195410183a69SYan, Zheng have |= POOL_READ; 195510183a69SYan, Zheng else if (err != -EPERM) 195610183a69SYan, Zheng goto out_unlock; 195710183a69SYan, Zheng 195810183a69SYan, Zheng if (err2 == 0 || err2 == -EEXIST) 195910183a69SYan, Zheng have |= POOL_WRITE; 196010183a69SYan, Zheng else if (err2 != -EPERM) { 196110183a69SYan, Zheng err = err2; 196210183a69SYan, Zheng goto out_unlock; 196310183a69SYan, Zheng } 196410183a69SYan, Zheng 1965779fe0fbSYan, Zheng pool_ns_len = pool_ns ? pool_ns->len : 0; 1966779fe0fbSYan, Zheng perm = kmalloc(sizeof(*perm) + pool_ns_len + 1, GFP_NOFS); 196710183a69SYan, Zheng if (!perm) { 196810183a69SYan, Zheng err = -ENOMEM; 196910183a69SYan, Zheng goto out_unlock; 197010183a69SYan, Zheng } 197110183a69SYan, Zheng 197210183a69SYan, Zheng perm->pool = pool; 197310183a69SYan, Zheng perm->perm = have; 1974779fe0fbSYan, Zheng perm->pool_ns_len = pool_ns_len; 1975779fe0fbSYan, Zheng if (pool_ns_len > 0) 1976779fe0fbSYan, Zheng memcpy(perm->pool_ns, pool_ns->str, pool_ns_len); 1977779fe0fbSYan, Zheng perm->pool_ns[pool_ns_len] = 0; 1978779fe0fbSYan, Zheng 197910183a69SYan, Zheng rb_link_node(&perm->node, parent, p); 198010183a69SYan, Zheng rb_insert_color(&perm->node, &mdsc->pool_perm_tree); 198110183a69SYan, Zheng err = 0; 198210183a69SYan, Zheng out_unlock: 198310183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 198410183a69SYan, Zheng 198510183a69SYan, Zheng ceph_osdc_put_request(rd_req); 198610183a69SYan, Zheng ceph_osdc_put_request(wr_req); 198710183a69SYan, Zheng out: 198810183a69SYan, Zheng if (!err) 198910183a69SYan, Zheng err = have; 1990779fe0fbSYan, Zheng if (pool_ns) 1991779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s result = %d\n", 1992779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str, err); 1993779fe0fbSYan, Zheng else 19947627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld result = %d\n", pool, err); 199510183a69SYan, Zheng return err; 199610183a69SYan, Zheng } 199710183a69SYan, Zheng 199810183a69SYan, Zheng int ceph_pool_perm_check(struct ceph_inode_info *ci, int need) 199910183a69SYan, Zheng { 20007627151eSYan, Zheng s64 pool; 2001779fe0fbSYan, Zheng struct ceph_string *pool_ns; 200210183a69SYan, Zheng int ret, flags; 200310183a69SYan, Zheng 200480e80fbbSYan, Zheng if (ci->i_vino.snap != CEPH_NOSNAP) { 200580e80fbbSYan, Zheng /* 200680e80fbbSYan, Zheng * Pool permission check needs to write to the first object. 200780e80fbbSYan, Zheng * But for snapshot, head of the first object may have alread 200880e80fbbSYan, Zheng * been deleted. Skip check to avoid creating orphan object. 200980e80fbbSYan, Zheng */ 201080e80fbbSYan, Zheng return 0; 201180e80fbbSYan, Zheng } 201280e80fbbSYan, Zheng 201310183a69SYan, Zheng if (ceph_test_mount_opt(ceph_inode_to_client(&ci->vfs_inode), 201410183a69SYan, Zheng NOPOOLPERM)) 201510183a69SYan, Zheng return 0; 201610183a69SYan, Zheng 201710183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 201810183a69SYan, Zheng flags = ci->i_ceph_flags; 20197627151eSYan, Zheng pool = ci->i_layout.pool_id; 202010183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 202110183a69SYan, Zheng check: 202210183a69SYan, Zheng if (flags & CEPH_I_POOL_PERM) { 202310183a69SYan, Zheng if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { 20247627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no read perm\n", 202510183a69SYan, Zheng pool); 202610183a69SYan, Zheng return -EPERM; 202710183a69SYan, Zheng } 202810183a69SYan, Zheng if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { 20297627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no write perm\n", 203010183a69SYan, Zheng pool); 203110183a69SYan, Zheng return -EPERM; 203210183a69SYan, Zheng } 203310183a69SYan, Zheng return 0; 203410183a69SYan, Zheng } 203510183a69SYan, Zheng 2036779fe0fbSYan, Zheng pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); 2037779fe0fbSYan, Zheng ret = __ceph_pool_perm_get(ci, pool, pool_ns); 2038779fe0fbSYan, Zheng ceph_put_string(pool_ns); 203910183a69SYan, Zheng if (ret < 0) 204010183a69SYan, Zheng return ret; 204110183a69SYan, Zheng 204210183a69SYan, Zheng flags = CEPH_I_POOL_PERM; 204310183a69SYan, Zheng if (ret & POOL_READ) 204410183a69SYan, Zheng flags |= CEPH_I_POOL_RD; 204510183a69SYan, Zheng if (ret & POOL_WRITE) 204610183a69SYan, Zheng flags |= CEPH_I_POOL_WR; 204710183a69SYan, Zheng 204810183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 2049779fe0fbSYan, Zheng if (pool == ci->i_layout.pool_id && 2050779fe0fbSYan, Zheng pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { 2051779fe0fbSYan, Zheng ci->i_ceph_flags |= flags; 205210183a69SYan, Zheng } else { 20537627151eSYan, Zheng pool = ci->i_layout.pool_id; 205410183a69SYan, Zheng flags = ci->i_ceph_flags; 205510183a69SYan, Zheng } 205610183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 205710183a69SYan, Zheng goto check; 205810183a69SYan, Zheng } 205910183a69SYan, Zheng 206010183a69SYan, Zheng void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) 206110183a69SYan, Zheng { 206210183a69SYan, Zheng struct ceph_pool_perm *perm; 206310183a69SYan, Zheng struct rb_node *n; 206410183a69SYan, Zheng 206510183a69SYan, Zheng while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { 206610183a69SYan, Zheng n = rb_first(&mdsc->pool_perm_tree); 206710183a69SYan, Zheng perm = rb_entry(n, struct ceph_pool_perm, node); 206810183a69SYan, Zheng rb_erase(n, &mdsc->pool_perm_tree); 206910183a69SYan, Zheng kfree(perm); 207010183a69SYan, Zheng } 207110183a69SYan, Zheng } 2072