13d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 21d3576fdSSage Weil 31d3576fdSSage Weil #include <linux/backing-dev.h> 41d3576fdSSage Weil #include <linux/fs.h> 51d3576fdSSage Weil #include <linux/mm.h> 61d3576fdSSage Weil #include <linux/pagemap.h> 71d3576fdSSage Weil #include <linux/writeback.h> /* generic_writepages */ 85a0e3ad6STejun Heo #include <linux/slab.h> 91d3576fdSSage Weil #include <linux/pagevec.h> 101d3576fdSSage Weil #include <linux/task_io_accounting_ops.h> 111d3576fdSSage Weil 121d3576fdSSage Weil #include "super.h" 133d14c5d2SYehuda Sadeh #include "mds_client.h" 1499ccbd22SMilosz Tanski #include "cache.h" 153d14c5d2SYehuda Sadeh #include <linux/ceph/osd_client.h> 161d3576fdSSage Weil 171d3576fdSSage Weil /* 181d3576fdSSage Weil * Ceph address space ops. 191d3576fdSSage Weil * 201d3576fdSSage Weil * There are a few funny things going on here. 211d3576fdSSage Weil * 221d3576fdSSage Weil * The page->private field is used to reference a struct 231d3576fdSSage Weil * ceph_snap_context for _every_ dirty page. This indicates which 241d3576fdSSage Weil * snapshot the page was logically dirtied in, and thus which snap 251d3576fdSSage Weil * context needs to be associated with the osd write during writeback. 261d3576fdSSage Weil * 271d3576fdSSage Weil * Similarly, struct ceph_inode_info maintains a set of counters to 2825985edcSLucas De Marchi * count dirty pages on the inode. In the absence of snapshots, 291d3576fdSSage Weil * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. 301d3576fdSSage Weil * 311d3576fdSSage Weil * When a snapshot is taken (that is, when the client receives 321d3576fdSSage Weil * notification that a snapshot was taken), each inode with caps and 331d3576fdSSage Weil * with dirty pages (dirty pages implies there is a cap) gets a new 341d3576fdSSage Weil * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending 351d3576fdSSage Weil * order, new snaps go to the tail). The i_wrbuffer_ref_head count is 361d3576fdSSage Weil * moved to capsnap->dirty. (Unless a sync write is currently in 371d3576fdSSage Weil * progress. In that case, the capsnap is said to be "pending", new 381d3576fdSSage Weil * writes cannot start, and the capsnap isn't "finalized" until the 391d3576fdSSage Weil * write completes (or fails) and a final size/mtime for the inode for 401d3576fdSSage Weil * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. 411d3576fdSSage Weil * 421d3576fdSSage Weil * On writeback, we must submit writes to the osd IN SNAP ORDER. So, 431d3576fdSSage Weil * we look for the first capsnap in i_cap_snaps and write out pages in 441d3576fdSSage Weil * that snap context _only_. Then we move on to the next capsnap, 451d3576fdSSage Weil * eventually reaching the "live" or "head" context (i.e., pages that 461d3576fdSSage Weil * are not yet snapped) and are writing the most recently dirtied 471d3576fdSSage Weil * pages. 481d3576fdSSage Weil * 491d3576fdSSage Weil * Invalidate and so forth must take care to ensure the dirty page 501d3576fdSSage Weil * accounting is preserved. 511d3576fdSSage Weil */ 521d3576fdSSage Weil 532baba250SYehuda Sadeh #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) 542baba250SYehuda Sadeh #define CONGESTION_OFF_THRESH(congestion_kb) \ 552baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) - \ 562baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) >> 2)) 572baba250SYehuda Sadeh 5861600ef8SYan, Zheng static inline struct ceph_snap_context *page_snap_context(struct page *page) 5961600ef8SYan, Zheng { 6061600ef8SYan, Zheng if (PagePrivate(page)) 6161600ef8SYan, Zheng return (void *)page->private; 6261600ef8SYan, Zheng return NULL; 6361600ef8SYan, Zheng } 641d3576fdSSage Weil 651d3576fdSSage Weil /* 661d3576fdSSage Weil * Dirty a page. Optimistically adjust accounting, on the assumption 671d3576fdSSage Weil * that we won't race with invalidate. If we do, readjust. 681d3576fdSSage Weil */ 691d3576fdSSage Weil static int ceph_set_page_dirty(struct page *page) 701d3576fdSSage Weil { 711d3576fdSSage Weil struct address_space *mapping = page->mapping; 721d3576fdSSage Weil struct inode *inode; 731d3576fdSSage Weil struct ceph_inode_info *ci; 741d3576fdSSage Weil struct ceph_snap_context *snapc; 757d6e1f54SSha Zhengju int ret; 761d3576fdSSage Weil 771d3576fdSSage Weil if (unlikely(!mapping)) 781d3576fdSSage Weil return !TestSetPageDirty(page); 791d3576fdSSage Weil 807d6e1f54SSha Zhengju if (PageDirty(page)) { 811d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu -- already dirty\n", 821d3576fdSSage Weil mapping->host, page, page->index); 837d6e1f54SSha Zhengju BUG_ON(!PagePrivate(page)); 841d3576fdSSage Weil return 0; 851d3576fdSSage Weil } 861d3576fdSSage Weil 871d3576fdSSage Weil inode = mapping->host; 881d3576fdSSage Weil ci = ceph_inode(inode); 891d3576fdSSage Weil 901d3576fdSSage Weil /* dirty the head */ 91be655596SSage Weil spin_lock(&ci->i_ceph_lock); 925dda377cSYan, Zheng BUG_ON(ci->i_wr_ref == 0); // caller should hold Fw reference 935dda377cSYan, Zheng if (__ceph_have_pending_cap_snap(ci)) { 945dda377cSYan, Zheng struct ceph_cap_snap *capsnap = 955dda377cSYan, Zheng list_last_entry(&ci->i_cap_snaps, 965dda377cSYan, Zheng struct ceph_cap_snap, 975dda377cSYan, Zheng ci_item); 985dda377cSYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 995dda377cSYan, Zheng capsnap->dirty_pages++; 1005dda377cSYan, Zheng } else { 1015dda377cSYan, Zheng BUG_ON(!ci->i_head_snapc); 1025dda377cSYan, Zheng snapc = ceph_get_snap_context(ci->i_head_snapc); 1031d3576fdSSage Weil ++ci->i_wrbuffer_ref_head; 1045dda377cSYan, Zheng } 1051d3576fdSSage Weil if (ci->i_wrbuffer_ref == 0) 1060444d76aSDave Chinner ihold(inode); 1071d3576fdSSage Weil ++ci->i_wrbuffer_ref; 1081d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu head %d/%d -> %d/%d " 1091d3576fdSSage Weil "snapc %p seq %lld (%d snaps)\n", 1101d3576fdSSage Weil mapping->host, page, page->index, 1111d3576fdSSage Weil ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, 1121d3576fdSSage Weil ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, 1131d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 114be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 1151d3576fdSSage Weil 1161d3576fdSSage Weil /* 1171d3576fdSSage Weil * Reference snap context in page->private. Also set 1181d3576fdSSage Weil * PagePrivate so that we get invalidatepage callback. 1191d3576fdSSage Weil */ 1207d6e1f54SSha Zhengju BUG_ON(PagePrivate(page)); 1211d3576fdSSage Weil page->private = (unsigned long)snapc; 1221d3576fdSSage Weil SetPagePrivate(page); 1231d3576fdSSage Weil 1247d6e1f54SSha Zhengju ret = __set_page_dirty_nobuffers(page); 1257d6e1f54SSha Zhengju WARN_ON(!PageLocked(page)); 1267d6e1f54SSha Zhengju WARN_ON(!page->mapping); 1271d3576fdSSage Weil 1287d6e1f54SSha Zhengju return ret; 1291d3576fdSSage Weil } 1301d3576fdSSage Weil 1311d3576fdSSage Weil /* 1321d3576fdSSage Weil * If we are truncating the full page (i.e. offset == 0), adjust the 1331d3576fdSSage Weil * dirty page counters appropriately. Only called if there is private 1341d3576fdSSage Weil * data on the page. 1351d3576fdSSage Weil */ 136d47992f8SLukas Czerner static void ceph_invalidatepage(struct page *page, unsigned int offset, 137d47992f8SLukas Czerner unsigned int length) 1381d3576fdSSage Weil { 1394ce1e9adSAlexander Beregalov struct inode *inode; 1401d3576fdSSage Weil struct ceph_inode_info *ci; 14161600ef8SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 1421d3576fdSSage Weil 1434ce1e9adSAlexander Beregalov inode = page->mapping->host; 144b150f5c1SMilosz Tanski ci = ceph_inode(inode); 145b150f5c1SMilosz Tanski 146b150f5c1SMilosz Tanski if (offset != 0 || length != PAGE_CACHE_SIZE) { 147b150f5c1SMilosz Tanski dout("%p invalidatepage %p idx %lu partial dirty page %u~%u\n", 148b150f5c1SMilosz Tanski inode, page, page->index, offset, length); 149b150f5c1SMilosz Tanski return; 150b150f5c1SMilosz Tanski } 1514ce1e9adSAlexander Beregalov 15299ccbd22SMilosz Tanski ceph_invalidate_fscache_page(inode, page); 15399ccbd22SMilosz Tanski 15499ccbd22SMilosz Tanski if (!PagePrivate(page)) 15599ccbd22SMilosz Tanski return; 15699ccbd22SMilosz Tanski 1571d3576fdSSage Weil /* 1581d3576fdSSage Weil * We can get non-dirty pages here due to races between 1591d3576fdSSage Weil * set_page_dirty and truncate_complete_page; just spit out a 1601d3576fdSSage Weil * warning, in case we end up with accounting problems later. 1611d3576fdSSage Weil */ 1621d3576fdSSage Weil if (!PageDirty(page)) 1631d3576fdSSage Weil pr_err("%p invalidatepage %p page not dirty\n", inode, page); 1641d3576fdSSage Weil 1651d3576fdSSage Weil ClearPageChecked(page); 1661d3576fdSSage Weil 167569d39fcSLukas Czerner dout("%p invalidatepage %p idx %lu full dirty page\n", 168569d39fcSLukas Czerner inode, page, page->index); 169b150f5c1SMilosz Tanski 1701d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 1711d3576fdSSage Weil ceph_put_snap_context(snapc); 1721d3576fdSSage Weil page->private = 0; 1731d3576fdSSage Weil ClearPagePrivate(page); 1741d3576fdSSage Weil } 1751d3576fdSSage Weil 1761d3576fdSSage Weil static int ceph_releasepage(struct page *page, gfp_t g) 1771d3576fdSSage Weil { 1781d3576fdSSage Weil struct inode *inode = page->mapping ? page->mapping->host : NULL; 1791d3576fdSSage Weil dout("%p releasepage %p idx %lu\n", inode, page, page->index); 1801d3576fdSSage Weil WARN_ON(PageDirty(page)); 18199ccbd22SMilosz Tanski 18299ccbd22SMilosz Tanski /* Can we release the page from the cache? */ 18399ccbd22SMilosz Tanski if (!ceph_release_fscache_page(page, g)) 1841d3576fdSSage Weil return 0; 18599ccbd22SMilosz Tanski 18699ccbd22SMilosz Tanski return !PagePrivate(page); 1871d3576fdSSage Weil } 1881d3576fdSSage Weil 1891d3576fdSSage Weil /* 1901d3576fdSSage Weil * read a single page, without unlocking it. 1911d3576fdSSage Weil */ 1921d3576fdSSage Weil static int readpage_nounlock(struct file *filp, struct page *page) 1931d3576fdSSage Weil { 194496ad9aaSAl Viro struct inode *inode = file_inode(filp); 1951d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 1963d14c5d2SYehuda Sadeh struct ceph_osd_client *osdc = 1973d14c5d2SYehuda Sadeh &ceph_inode_to_client(inode)->client->osdc; 1981d3576fdSSage Weil int err = 0; 19983701246SYan, Zheng u64 off = page_offset(page); 2001d3576fdSSage Weil u64 len = PAGE_CACHE_SIZE; 2011d3576fdSSage Weil 20283701246SYan, Zheng if (off >= i_size_read(inode)) { 203fcc02d2aSYan, Zheng zero_user_segment(page, 0, PAGE_CACHE_SIZE); 20483701246SYan, Zheng SetPageUptodate(page); 20583701246SYan, Zheng return 0; 20683701246SYan, Zheng } 20799ccbd22SMilosz Tanski 208fcc02d2aSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 20983701246SYan, Zheng /* 210fcc02d2aSYan, Zheng * Uptodate inline data should have been added 211fcc02d2aSYan, Zheng * into page cache while getting Fcr caps. 21283701246SYan, Zheng */ 213fcc02d2aSYan, Zheng if (off == 0) 21483701246SYan, Zheng return -EINVAL; 215fcc02d2aSYan, Zheng zero_user_segment(page, 0, PAGE_CACHE_SIZE); 216fcc02d2aSYan, Zheng SetPageUptodate(page); 217fcc02d2aSYan, Zheng return 0; 218fcc02d2aSYan, Zheng } 21983701246SYan, Zheng 22083701246SYan, Zheng err = ceph_readpage_from_fscache(inode, page); 22199ccbd22SMilosz Tanski if (err == 0) 22299ccbd22SMilosz Tanski goto out; 22399ccbd22SMilosz Tanski 2241d3576fdSSage Weil dout("readpage inode %p file %p page %p index %lu\n", 2251d3576fdSSage Weil inode, filp, page, page->index); 2261d3576fdSSage Weil err = ceph_osdc_readpages(osdc, ceph_vino(inode), &ci->i_layout, 22783701246SYan, Zheng off, &len, 2281d3576fdSSage Weil ci->i_truncate_seq, ci->i_truncate_size, 229b7495fc2SSage Weil &page, 1, 0); 2301d3576fdSSage Weil if (err == -ENOENT) 2311d3576fdSSage Weil err = 0; 2321d3576fdSSage Weil if (err < 0) { 2331d3576fdSSage Weil SetPageError(page); 23418302805SLi Wang ceph_fscache_readpage_cancel(inode, page); 2351d3576fdSSage Weil goto out; 23623cd573bSZhang Zhen } 23723cd573bSZhang Zhen if (err < PAGE_CACHE_SIZE) 2381d3576fdSSage Weil /* zero fill remainder of page */ 2391d3576fdSSage Weil zero_user_segment(page, err, PAGE_CACHE_SIZE); 24023cd573bSZhang Zhen else 24156f91aadSLi Wang flush_dcache_page(page); 2421d3576fdSSage Weil 24323cd573bSZhang Zhen SetPageUptodate(page); 24499ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 24599ccbd22SMilosz Tanski 2461d3576fdSSage Weil out: 2471d3576fdSSage Weil return err < 0 ? err : 0; 2481d3576fdSSage Weil } 2491d3576fdSSage Weil 2501d3576fdSSage Weil static int ceph_readpage(struct file *filp, struct page *page) 2511d3576fdSSage Weil { 2521d3576fdSSage Weil int r = readpage_nounlock(filp, page); 2531d3576fdSSage Weil unlock_page(page); 2541d3576fdSSage Weil return r; 2551d3576fdSSage Weil } 2561d3576fdSSage Weil 2571d3576fdSSage Weil /* 2587c272194SSage Weil * Finish an async read(ahead) op. 2591d3576fdSSage Weil */ 2607c272194SSage Weil static void finish_read(struct ceph_osd_request *req, struct ceph_msg *msg) 2611d3576fdSSage Weil { 2627c272194SSage Weil struct inode *inode = req->r_inode; 26387060c10SAlex Elder struct ceph_osd_data *osd_data; 2641b83bef2SSage Weil int rc = req->r_result; 2651b83bef2SSage Weil int bytes = le32_to_cpu(msg->hdr.data_len); 266e0c59487SAlex Elder int num_pages; 2677c272194SSage Weil int i; 2687c272194SSage Weil 2697c272194SSage Weil dout("finish_read %p req %p rc %d bytes %d\n", inode, req, rc, bytes); 2707c272194SSage Weil 2717c272194SSage Weil /* unlock all pages, zeroing any data we didn't read */ 272406e2c9fSAlex Elder osd_data = osd_req_op_extent_osd_data(req, 0); 27387060c10SAlex Elder BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 27487060c10SAlex Elder num_pages = calc_pages_for((u64)osd_data->alignment, 27587060c10SAlex Elder (u64)osd_data->length); 276e0c59487SAlex Elder for (i = 0; i < num_pages; i++) { 27787060c10SAlex Elder struct page *page = osd_data->pages[i]; 2787c272194SSage Weil 27943838685SYan, Zheng if (rc < 0 && rc != ENOENT) 280f36132a7SLi Wang goto unlock; 2817c272194SSage Weil if (bytes < (int)PAGE_CACHE_SIZE) { 2827c272194SSage Weil /* zero (remainder of) page */ 2837c272194SSage Weil int s = bytes < 0 ? 0 : bytes; 2847c272194SSage Weil zero_user_segment(page, s, PAGE_CACHE_SIZE); 2857c272194SSage Weil } 2867c272194SSage Weil dout("finish_read %p uptodate %p idx %lu\n", inode, page, 2877c272194SSage Weil page->index); 2887c272194SSage Weil flush_dcache_page(page); 2897c272194SSage Weil SetPageUptodate(page); 29099ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 291f36132a7SLi Wang unlock: 2927c272194SSage Weil unlock_page(page); 2937c272194SSage Weil page_cache_release(page); 2940fff87ecSAlex Elder bytes -= PAGE_CACHE_SIZE; 2957c272194SSage Weil } 29687060c10SAlex Elder kfree(osd_data->pages); 2977c272194SSage Weil } 2987c272194SSage Weil 2998884d53dSDavid Zafman static void ceph_unlock_page_vector(struct page **pages, int num_pages) 3008884d53dSDavid Zafman { 3018884d53dSDavid Zafman int i; 3028884d53dSDavid Zafman 3038884d53dSDavid Zafman for (i = 0; i < num_pages; i++) 3048884d53dSDavid Zafman unlock_page(pages[i]); 3058884d53dSDavid Zafman } 3068884d53dSDavid Zafman 3077c272194SSage Weil /* 3087c272194SSage Weil * start an async read(ahead) operation. return nr_pages we submitted 3097c272194SSage Weil * a read for on success, or negative error code. 3107c272194SSage Weil */ 3110d66a487SSage Weil static int start_read(struct inode *inode, struct list_head *page_list, int max) 3127c272194SSage Weil { 3137c272194SSage Weil struct ceph_osd_client *osdc = 3147c272194SSage Weil &ceph_inode_to_client(inode)->client->osdc; 3157c272194SSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 3167c272194SSage Weil struct page *page = list_entry(page_list->prev, struct page, lru); 317acead002SAlex Elder struct ceph_vino vino; 3187c272194SSage Weil struct ceph_osd_request *req; 3197c272194SSage Weil u64 off; 3207c272194SSage Weil u64 len; 3217c272194SSage Weil int i; 3221d3576fdSSage Weil struct page **pages; 3237c272194SSage Weil pgoff_t next_index; 3247c272194SSage Weil int nr_pages = 0; 3257c272194SSage Weil int ret; 3267c272194SSage Weil 3276285bc23SAlex Elder off = (u64) page_offset(page); 3287c272194SSage Weil 3297c272194SSage Weil /* count pages */ 3307c272194SSage Weil next_index = page->index; 3317c272194SSage Weil list_for_each_entry_reverse(page, page_list, lru) { 3327c272194SSage Weil if (page->index != next_index) 3337c272194SSage Weil break; 3347c272194SSage Weil nr_pages++; 3357c272194SSage Weil next_index++; 3360d66a487SSage Weil if (max && nr_pages == max) 3370d66a487SSage Weil break; 3387c272194SSage Weil } 3397c272194SSage Weil len = nr_pages << PAGE_CACHE_SHIFT; 3407c272194SSage Weil dout("start_read %p nr_pages %d is %lld~%lld\n", inode, nr_pages, 3417c272194SSage Weil off, len); 342acead002SAlex Elder vino = ceph_vino(inode); 343acead002SAlex Elder req = ceph_osdc_new_request(osdc, &ci->i_layout, vino, off, &len, 344715e4cd4SYan, Zheng 0, 1, CEPH_OSD_OP_READ, 345acead002SAlex Elder CEPH_OSD_FLAG_READ, NULL, 3467c272194SSage Weil ci->i_truncate_seq, ci->i_truncate_size, 347acead002SAlex Elder false); 3486816282dSSage Weil if (IS_ERR(req)) 3496816282dSSage Weil return PTR_ERR(req); 3501d3576fdSSage Weil 3511d3576fdSSage Weil /* build page vector */ 352cf7b7e14SAlex Elder nr_pages = calc_pages_for(0, len); 353687265e5SYan, Zheng pages = kmalloc(sizeof(*pages) * nr_pages, GFP_KERNEL); 3547c272194SSage Weil ret = -ENOMEM; 3551d3576fdSSage Weil if (!pages) 3567c272194SSage Weil goto out; 3577c272194SSage Weil for (i = 0; i < nr_pages; ++i) { 3587c272194SSage Weil page = list_entry(page_list->prev, struct page, lru); 3597c272194SSage Weil BUG_ON(PageLocked(page)); 3607c272194SSage Weil list_del(&page->lru); 3611d3576fdSSage Weil 3627c272194SSage Weil dout("start_read %p adding %p idx %lu\n", inode, page, 3637c272194SSage Weil page->index); 3647c272194SSage Weil if (add_to_page_cache_lru(page, &inode->i_data, page->index, 365687265e5SYan, Zheng GFP_KERNEL)) { 366d4d3aa38SMilosz Tanski ceph_fscache_uncache_page(inode, page); 3677c272194SSage Weil page_cache_release(page); 3687c272194SSage Weil dout("start_read %p add_to_page_cache failed %p\n", 3697c272194SSage Weil inode, page); 3707c272194SSage Weil nr_pages = i; 3717c272194SSage Weil goto out_pages; 3721d3576fdSSage Weil } 3737c272194SSage Weil pages[i] = page; 3741d3576fdSSage Weil } 375406e2c9fSAlex Elder osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false); 3767c272194SSage Weil req->r_callback = finish_read; 3777c272194SSage Weil req->r_inode = inode; 3787c272194SSage Weil 37979528734SAlex Elder ceph_osdc_build_request(req, off, NULL, vino.snap, NULL); 38002ee07d3SAlex Elder 3817c272194SSage Weil dout("start_read %p starting %p %lld~%lld\n", inode, req, off, len); 3827c272194SSage Weil ret = ceph_osdc_start_request(osdc, req, false); 3837c272194SSage Weil if (ret < 0) 3847c272194SSage Weil goto out_pages; 3857c272194SSage Weil ceph_osdc_put_request(req); 3867c272194SSage Weil return nr_pages; 3877c272194SSage Weil 3887c272194SSage Weil out_pages: 3898884d53dSDavid Zafman ceph_unlock_page_vector(pages, nr_pages); 3907c272194SSage Weil ceph_release_page_vector(pages, nr_pages); 3917c272194SSage Weil out: 3927c272194SSage Weil ceph_osdc_put_request(req); 3937c272194SSage Weil return ret; 3941d3576fdSSage Weil } 3951d3576fdSSage Weil 3967c272194SSage Weil 3971d3576fdSSage Weil /* 3981d3576fdSSage Weil * Read multiple pages. Leave pages we don't read + unlock in page_list; 3991d3576fdSSage Weil * the caller (VM) cleans them up. 4001d3576fdSSage Weil */ 4011d3576fdSSage Weil static int ceph_readpages(struct file *file, struct address_space *mapping, 4021d3576fdSSage Weil struct list_head *page_list, unsigned nr_pages) 4031d3576fdSSage Weil { 404496ad9aaSAl Viro struct inode *inode = file_inode(file); 4050d66a487SSage Weil struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 4061d3576fdSSage Weil int rc = 0; 4070d66a487SSage Weil int max = 0; 4081d3576fdSSage Weil 40983701246SYan, Zheng if (ceph_inode(inode)->i_inline_version != CEPH_INLINE_NONE) 41083701246SYan, Zheng return -EINVAL; 41183701246SYan, Zheng 41299ccbd22SMilosz Tanski rc = ceph_readpages_from_fscache(mapping->host, mapping, page_list, 41399ccbd22SMilosz Tanski &nr_pages); 41499ccbd22SMilosz Tanski 41599ccbd22SMilosz Tanski if (rc == 0) 41699ccbd22SMilosz Tanski goto out; 41799ccbd22SMilosz Tanski 4180d66a487SSage Weil if (fsc->mount_options->rsize >= PAGE_CACHE_SIZE) 4190d66a487SSage Weil max = (fsc->mount_options->rsize + PAGE_CACHE_SIZE - 1) 4200d66a487SSage Weil >> PAGE_SHIFT; 4210d66a487SSage Weil 4222794a82aSAlex Elder dout("readpages %p file %p nr_pages %d max %d\n", inode, 4232794a82aSAlex Elder file, nr_pages, 4240d66a487SSage Weil max); 4257c272194SSage Weil while (!list_empty(page_list)) { 4260d66a487SSage Weil rc = start_read(inode, page_list, max); 4271d3576fdSSage Weil if (rc < 0) 4281d3576fdSSage Weil goto out; 4297c272194SSage Weil BUG_ON(rc == 0); 4301d3576fdSSage Weil } 4311d3576fdSSage Weil out: 43276be778bSMilosz Tanski ceph_fscache_readpages_cancel(inode, page_list); 43376be778bSMilosz Tanski 4347c272194SSage Weil dout("readpages %p file %p ret %d\n", inode, file, rc); 4351d3576fdSSage Weil return rc; 4361d3576fdSSage Weil } 4371d3576fdSSage Weil 4381d3576fdSSage Weil /* 4391d3576fdSSage Weil * Get ref for the oldest snapc for an inode with dirty data... that is, the 4401d3576fdSSage Weil * only snap context we are allowed to write back. 4411d3576fdSSage Weil */ 4426298a337SSage Weil static struct ceph_snap_context *get_oldest_context(struct inode *inode, 443e1966b49SYan, Zheng loff_t *snap_size) 4441d3576fdSSage Weil { 4451d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 4461d3576fdSSage Weil struct ceph_snap_context *snapc = NULL; 4471d3576fdSSage Weil struct ceph_cap_snap *capsnap = NULL; 4481d3576fdSSage Weil 449be655596SSage Weil spin_lock(&ci->i_ceph_lock); 4501d3576fdSSage Weil list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 4511d3576fdSSage Weil dout(" cap_snap %p snapc %p has %d dirty pages\n", capsnap, 4521d3576fdSSage Weil capsnap->context, capsnap->dirty_pages); 4531d3576fdSSage Weil if (capsnap->dirty_pages) { 4541d3576fdSSage Weil snapc = ceph_get_snap_context(capsnap->context); 4551d3576fdSSage Weil if (snap_size) 4561d3576fdSSage Weil *snap_size = capsnap->size; 4571d3576fdSSage Weil break; 4581d3576fdSSage Weil } 4591d3576fdSSage Weil } 4607d8cb26dSSage Weil if (!snapc && ci->i_wrbuffer_ref_head) { 46180e755feSSage Weil snapc = ceph_get_snap_context(ci->i_head_snapc); 4621d3576fdSSage Weil dout(" head snapc %p has %d dirty pages\n", 4631d3576fdSSage Weil snapc, ci->i_wrbuffer_ref_head); 4641d3576fdSSage Weil } 465be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 4661d3576fdSSage Weil return snapc; 4671d3576fdSSage Weil } 4681d3576fdSSage Weil 4691d3576fdSSage Weil /* 4701d3576fdSSage Weil * Write a single page, but leave the page locked. 4711d3576fdSSage Weil * 4721d3576fdSSage Weil * If we get a write error, set the page error bit, but still adjust the 4731d3576fdSSage Weil * dirty page accounting (i.e., page is no longer dirty). 4741d3576fdSSage Weil */ 4751d3576fdSSage Weil static int writepage_nounlock(struct page *page, struct writeback_control *wbc) 4761d3576fdSSage Weil { 4771d3576fdSSage Weil struct inode *inode; 4781d3576fdSSage Weil struct ceph_inode_info *ci; 4793d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc; 4801d3576fdSSage Weil struct ceph_osd_client *osdc; 4816298a337SSage Weil struct ceph_snap_context *snapc, *oldest; 482fc2744aaSYan, Zheng loff_t page_off = page_offset(page); 483e1966b49SYan, Zheng loff_t snap_size = -1; 4842baba250SYehuda Sadeh long writeback_stat; 485e1966b49SYan, Zheng u64 truncate_size; 486fc2744aaSYan, Zheng u32 truncate_seq; 487fc2744aaSYan, Zheng int err = 0, len = PAGE_CACHE_SIZE; 4881d3576fdSSage Weil 4891d3576fdSSage Weil dout("writepage %p idx %lu\n", page, page->index); 4901d3576fdSSage Weil 4911d3576fdSSage Weil if (!page->mapping || !page->mapping->host) { 4921d3576fdSSage Weil dout("writepage %p - no mapping\n", page); 4931d3576fdSSage Weil return -EFAULT; 4941d3576fdSSage Weil } 4951d3576fdSSage Weil inode = page->mapping->host; 4961d3576fdSSage Weil ci = ceph_inode(inode); 4973d14c5d2SYehuda Sadeh fsc = ceph_inode_to_client(inode); 4983d14c5d2SYehuda Sadeh osdc = &fsc->client->osdc; 4991d3576fdSSage Weil 5001d3576fdSSage Weil /* verify this is a writeable snap context */ 50161600ef8SYan, Zheng snapc = page_snap_context(page); 5021d3576fdSSage Weil if (snapc == NULL) { 5031d3576fdSSage Weil dout("writepage %p page %p not dirty?\n", inode, page); 5041d3576fdSSage Weil goto out; 5051d3576fdSSage Weil } 5066298a337SSage Weil oldest = get_oldest_context(inode, &snap_size); 5076298a337SSage Weil if (snapc->seq > oldest->seq) { 5081d3576fdSSage Weil dout("writepage %p page %p snapc %p not writeable - noop\n", 50961600ef8SYan, Zheng inode, page, snapc); 5101d3576fdSSage Weil /* we should only noop if called by kswapd */ 5111d3576fdSSage Weil WARN_ON((current->flags & PF_MEMALLOC) == 0); 5126298a337SSage Weil ceph_put_snap_context(oldest); 5131d3576fdSSage Weil goto out; 5141d3576fdSSage Weil } 5156298a337SSage Weil ceph_put_snap_context(oldest); 5161d3576fdSSage Weil 517fc2744aaSYan, Zheng spin_lock(&ci->i_ceph_lock); 518fc2744aaSYan, Zheng truncate_seq = ci->i_truncate_seq; 519fc2744aaSYan, Zheng truncate_size = ci->i_truncate_size; 520e1966b49SYan, Zheng if (snap_size == -1) 521fc2744aaSYan, Zheng snap_size = i_size_read(inode); 522fc2744aaSYan, Zheng spin_unlock(&ci->i_ceph_lock); 523fc2744aaSYan, Zheng 5241d3576fdSSage Weil /* is this a partial page at end of file? */ 525fc2744aaSYan, Zheng if (page_off >= snap_size) { 526fc2744aaSYan, Zheng dout("%p page eof %llu\n", page, snap_size); 527fc2744aaSYan, Zheng goto out; 528fc2744aaSYan, Zheng } 529fc2744aaSYan, Zheng if (snap_size < page_off + len) 530fc2744aaSYan, Zheng len = snap_size - page_off; 5311d3576fdSSage Weil 532ae00d4f3SSage Weil dout("writepage %p page %p index %lu on %llu~%u snapc %p\n", 533ae00d4f3SSage Weil inode, page, page->index, page_off, len, snapc); 5341d3576fdSSage Weil 5353d14c5d2SYehuda Sadeh writeback_stat = atomic_long_inc_return(&fsc->writeback_count); 5362baba250SYehuda Sadeh if (writeback_stat > 5373d14c5d2SYehuda Sadeh CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) 5383d14c5d2SYehuda Sadeh set_bdi_congested(&fsc->backing_dev_info, BLK_RW_ASYNC); 5392baba250SYehuda Sadeh 54099ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 54199ccbd22SMilosz Tanski 5421d3576fdSSage Weil set_page_writeback(page); 5431d3576fdSSage Weil err = ceph_osdc_writepages(osdc, ceph_vino(inode), 5441d3576fdSSage Weil &ci->i_layout, snapc, 5451d3576fdSSage Weil page_off, len, 546fc2744aaSYan, Zheng truncate_seq, truncate_size, 54724808826SAlex Elder &inode->i_mtime, &page, 1); 5481d3576fdSSage Weil if (err < 0) { 5491d3576fdSSage Weil dout("writepage setting page/mapping error %d %p\n", err, page); 5501d3576fdSSage Weil SetPageError(page); 5511d3576fdSSage Weil mapping_set_error(&inode->i_data, err); 5521d3576fdSSage Weil if (wbc) 5531d3576fdSSage Weil wbc->pages_skipped++; 5541d3576fdSSage Weil } else { 5551d3576fdSSage Weil dout("writepage cleaned page %p\n", page); 5561d3576fdSSage Weil err = 0; /* vfs expects us to return 0 */ 5571d3576fdSSage Weil } 5581d3576fdSSage Weil page->private = 0; 5591d3576fdSSage Weil ClearPagePrivate(page); 5601d3576fdSSage Weil end_page_writeback(page); 5611d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 5626298a337SSage Weil ceph_put_snap_context(snapc); /* page's reference */ 5631d3576fdSSage Weil out: 5641d3576fdSSage Weil return err; 5651d3576fdSSage Weil } 5661d3576fdSSage Weil 5671d3576fdSSage Weil static int ceph_writepage(struct page *page, struct writeback_control *wbc) 5681d3576fdSSage Weil { 569dbd646a8SYehuda Sadeh int err; 570dbd646a8SYehuda Sadeh struct inode *inode = page->mapping->host; 571dbd646a8SYehuda Sadeh BUG_ON(!inode); 57270b666c3SSage Weil ihold(inode); 573dbd646a8SYehuda Sadeh err = writepage_nounlock(page, wbc); 5741d3576fdSSage Weil unlock_page(page); 575dbd646a8SYehuda Sadeh iput(inode); 5761d3576fdSSage Weil return err; 5771d3576fdSSage Weil } 5781d3576fdSSage Weil 5791d3576fdSSage Weil 5801d3576fdSSage Weil /* 5811d3576fdSSage Weil * lame release_pages helper. release_pages() isn't exported to 5821d3576fdSSage Weil * modules. 5831d3576fdSSage Weil */ 5841d3576fdSSage Weil static void ceph_release_pages(struct page **pages, int num) 5851d3576fdSSage Weil { 5861d3576fdSSage Weil struct pagevec pvec; 5871d3576fdSSage Weil int i; 5881d3576fdSSage Weil 5891d3576fdSSage Weil pagevec_init(&pvec, 0); 5901d3576fdSSage Weil for (i = 0; i < num; i++) { 5911d3576fdSSage Weil if (pagevec_add(&pvec, pages[i]) == 0) 5921d3576fdSSage Weil pagevec_release(&pvec); 5931d3576fdSSage Weil } 5941d3576fdSSage Weil pagevec_release(&pvec); 5951d3576fdSSage Weil } 5961d3576fdSSage Weil 5971d3576fdSSage Weil /* 5981d3576fdSSage Weil * async writeback completion handler. 5991d3576fdSSage Weil * 6001d3576fdSSage Weil * If we get an error, set the mapping error bit, but not the individual 6011d3576fdSSage Weil * page error bits. 6021d3576fdSSage Weil */ 6031d3576fdSSage Weil static void writepages_finish(struct ceph_osd_request *req, 6041d3576fdSSage Weil struct ceph_msg *msg) 6051d3576fdSSage Weil { 6061d3576fdSSage Weil struct inode *inode = req->r_inode; 6071d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 60887060c10SAlex Elder struct ceph_osd_data *osd_data; 6091d3576fdSSage Weil struct page *page; 6105b64640cSYan, Zheng int num_pages, total_pages = 0; 6115b64640cSYan, Zheng int i, j; 6125b64640cSYan, Zheng int rc = req->r_result; 6131d3576fdSSage Weil struct ceph_snap_context *snapc = req->r_snapc; 6141d3576fdSSage Weil struct address_space *mapping = inode->i_mapping; 6153d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 6165b64640cSYan, Zheng bool remove_page; 6171d3576fdSSage Weil 6185b64640cSYan, Zheng 6195b64640cSYan, Zheng dout("writepages_finish %p rc %d\n", inode, rc); 6205b64640cSYan, Zheng if (rc < 0) 6211d3576fdSSage Weil mapping_set_error(mapping, rc); 622e63dc5c7SYehuda Sadeh 623e63dc5c7SYehuda Sadeh /* 624e63dc5c7SYehuda Sadeh * We lost the cache cap, need to truncate the page before 625e63dc5c7SYehuda Sadeh * it is unlocked, otherwise we'd truncate it later in the 626e63dc5c7SYehuda Sadeh * page truncation thread, possibly losing some data that 627e63dc5c7SYehuda Sadeh * raced its way in 628e63dc5c7SYehuda Sadeh */ 6295b64640cSYan, Zheng remove_page = !(ceph_caps_issued(ci) & 6305b64640cSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); 6315b64640cSYan, Zheng 6325b64640cSYan, Zheng /* clean all pages */ 6335b64640cSYan, Zheng for (i = 0; i < req->r_num_ops; i++) { 6345b64640cSYan, Zheng if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) 6355b64640cSYan, Zheng break; 6365b64640cSYan, Zheng 6375b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, i); 6385b64640cSYan, Zheng BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 6395b64640cSYan, Zheng num_pages = calc_pages_for((u64)osd_data->alignment, 6405b64640cSYan, Zheng (u64)osd_data->length); 6415b64640cSYan, Zheng total_pages += num_pages; 6425b64640cSYan, Zheng for (j = 0; j < num_pages; j++) { 6435b64640cSYan, Zheng page = osd_data->pages[j]; 6445b64640cSYan, Zheng BUG_ON(!page); 6455b64640cSYan, Zheng WARN_ON(!PageUptodate(page)); 6465b64640cSYan, Zheng 6475b64640cSYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 6485b64640cSYan, Zheng CONGESTION_OFF_THRESH( 6495b64640cSYan, Zheng fsc->mount_options->congestion_kb)) 6505b64640cSYan, Zheng clear_bdi_congested(&fsc->backing_dev_info, 6515b64640cSYan, Zheng BLK_RW_ASYNC); 6525b64640cSYan, Zheng 6535b64640cSYan, Zheng ceph_put_snap_context(page_snap_context(page)); 6545b64640cSYan, Zheng page->private = 0; 6555b64640cSYan, Zheng ClearPagePrivate(page); 6565b64640cSYan, Zheng dout("unlocking %p\n", page); 6575b64640cSYan, Zheng end_page_writeback(page); 6585b64640cSYan, Zheng 6595b64640cSYan, Zheng if (remove_page) 6605b64640cSYan, Zheng generic_error_remove_page(inode->i_mapping, 6615b64640cSYan, Zheng page); 662e63dc5c7SYehuda Sadeh 6631d3576fdSSage Weil unlock_page(page); 6641d3576fdSSage Weil } 6655b64640cSYan, Zheng dout("writepages_finish %p wrote %llu bytes cleaned %d pages\n", 6665b64640cSYan, Zheng inode, osd_data->length, rc >= 0 ? num_pages : 0); 6671d3576fdSSage Weil 66887060c10SAlex Elder ceph_release_pages(osd_data->pages, num_pages); 6695b64640cSYan, Zheng } 6705b64640cSYan, Zheng 6715b64640cSYan, Zheng ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); 6725b64640cSYan, Zheng 6735b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, 0); 67487060c10SAlex Elder if (osd_data->pages_from_pool) 67587060c10SAlex Elder mempool_free(osd_data->pages, 676640ef79dSCheng Renquan ceph_sb_to_client(inode->i_sb)->wb_pagevec_pool); 6771d3576fdSSage Weil else 67887060c10SAlex Elder kfree(osd_data->pages); 6791d3576fdSSage Weil ceph_osdc_put_request(req); 6801d3576fdSSage Weil } 6811d3576fdSSage Weil 6821d3576fdSSage Weil /* 6831d3576fdSSage Weil * initiate async writeback 6841d3576fdSSage Weil */ 6851d3576fdSSage Weil static int ceph_writepages_start(struct address_space *mapping, 6861d3576fdSSage Weil struct writeback_control *wbc) 6871d3576fdSSage Weil { 6881d3576fdSSage Weil struct inode *inode = mapping->host; 6891d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 690fc2744aaSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 691fc2744aaSYan, Zheng struct ceph_vino vino = ceph_vino(inode); 6921d3576fdSSage Weil pgoff_t index, start, end; 6931d3576fdSSage Weil int range_whole = 0; 6941d3576fdSSage Weil int should_loop = 1; 6951d3576fdSSage Weil pgoff_t max_pages = 0, max_pages_ever = 0; 69680e755feSSage Weil struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; 6971d3576fdSSage Weil struct pagevec pvec; 6981d3576fdSSage Weil int done = 0; 6991d3576fdSSage Weil int rc = 0; 7001d3576fdSSage Weil unsigned wsize = 1 << inode->i_blkbits; 7011d3576fdSSage Weil struct ceph_osd_request *req = NULL; 702021b77beSDan Carpenter int do_sync = 0; 703e1966b49SYan, Zheng loff_t snap_size, i_size; 704e1966b49SYan, Zheng u64 truncate_size; 705fc2744aaSYan, Zheng u32 truncate_seq; 7061d3576fdSSage Weil 7071d3576fdSSage Weil /* 7081d3576fdSSage Weil * Include a 'sync' in the OSD request if this is a data 7091d3576fdSSage Weil * integrity write (e.g., O_SYNC write or fsync()), or if our 7101d3576fdSSage Weil * cap is being revoked. 7111d3576fdSSage Weil */ 712c62988ecSmajianpeng if ((wbc->sync_mode == WB_SYNC_ALL) || 713c62988ecSmajianpeng ceph_caps_revoking(ci, CEPH_CAP_FILE_BUFFER)) 7141d3576fdSSage Weil do_sync = 1; 7151d3576fdSSage Weil dout("writepages_start %p dosync=%d (mode=%s)\n", 7161d3576fdSSage Weil inode, do_sync, 7171d3576fdSSage Weil wbc->sync_mode == WB_SYNC_NONE ? "NONE" : 7181d3576fdSSage Weil (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); 7191d3576fdSSage Weil 72048fec5d0SYan, Zheng if (ACCESS_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { 721f3ae1b97SFabian Frederick pr_warn("writepage_start %p on forced umount\n", inode); 722a341d4dfSYan, Zheng truncate_pagecache(inode, 0); 723a341d4dfSYan, Zheng mapping_set_error(mapping, -EIO); 7241d3576fdSSage Weil return -EIO; /* we're in a forced umount, don't write! */ 7251d3576fdSSage Weil } 7263d14c5d2SYehuda Sadeh if (fsc->mount_options->wsize && fsc->mount_options->wsize < wsize) 7273d14c5d2SYehuda Sadeh wsize = fsc->mount_options->wsize; 7281d3576fdSSage Weil if (wsize < PAGE_CACHE_SIZE) 7291d3576fdSSage Weil wsize = PAGE_CACHE_SIZE; 7301d3576fdSSage Weil max_pages_ever = wsize >> PAGE_CACHE_SHIFT; 7311d3576fdSSage Weil 7321d3576fdSSage Weil pagevec_init(&pvec, 0); 7331d3576fdSSage Weil 7341d3576fdSSage Weil /* where to start/end? */ 7351d3576fdSSage Weil if (wbc->range_cyclic) { 7361d3576fdSSage Weil start = mapping->writeback_index; /* Start from prev offset */ 7371d3576fdSSage Weil end = -1; 7381d3576fdSSage Weil dout(" cyclic, start at %lu\n", start); 7391d3576fdSSage Weil } else { 7401d3576fdSSage Weil start = wbc->range_start >> PAGE_CACHE_SHIFT; 7411d3576fdSSage Weil end = wbc->range_end >> PAGE_CACHE_SHIFT; 7421d3576fdSSage Weil if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 7431d3576fdSSage Weil range_whole = 1; 7441d3576fdSSage Weil should_loop = 0; 7451d3576fdSSage Weil dout(" not cyclic, %lu to %lu\n", start, end); 7461d3576fdSSage Weil } 7471d3576fdSSage Weil index = start; 7481d3576fdSSage Weil 7491d3576fdSSage Weil retry: 7501d3576fdSSage Weil /* find oldest snap context with dirty data */ 7511d3576fdSSage Weil ceph_put_snap_context(snapc); 752e1966b49SYan, Zheng snap_size = -1; 7531d3576fdSSage Weil snapc = get_oldest_context(inode, &snap_size); 7541d3576fdSSage Weil if (!snapc) { 7551d3576fdSSage Weil /* hmm, why does writepages get called when there 7561d3576fdSSage Weil is no dirty data? */ 7571d3576fdSSage Weil dout(" no snap context with dirty data?\n"); 7581d3576fdSSage Weil goto out; 7591d3576fdSSage Weil } 7601d3576fdSSage Weil dout(" oldest snapc is %p seq %lld (%d snaps)\n", 7611d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 762fc2744aaSYan, Zheng 763fc2744aaSYan, Zheng spin_lock(&ci->i_ceph_lock); 764fc2744aaSYan, Zheng truncate_seq = ci->i_truncate_seq; 765fc2744aaSYan, Zheng truncate_size = ci->i_truncate_size; 766e1966b49SYan, Zheng i_size = i_size_read(inode); 767fc2744aaSYan, Zheng spin_unlock(&ci->i_ceph_lock); 768fc2744aaSYan, Zheng 7691d3576fdSSage Weil if (last_snapc && snapc != last_snapc) { 7701d3576fdSSage Weil /* if we switched to a newer snapc, restart our scan at the 7711d3576fdSSage Weil * start of the original file range. */ 7721d3576fdSSage Weil dout(" snapc differs from last pass, restarting at %lu\n", 7731d3576fdSSage Weil index); 7741d3576fdSSage Weil index = start; 7751d3576fdSSage Weil } 7761d3576fdSSage Weil last_snapc = snapc; 7771d3576fdSSage Weil 7781d3576fdSSage Weil while (!done && index <= end) { 7791d3576fdSSage Weil unsigned i; 7801d3576fdSSage Weil int first; 7815b64640cSYan, Zheng pgoff_t strip_unit_end = 0; 7825b64640cSYan, Zheng int num_ops = 0, op_idx; 7835b64640cSYan, Zheng int pvec_pages, locked_pages = 0; 7845b64640cSYan, Zheng struct page **pages = NULL, **data_pages; 785e5975c7cSAlex Elder mempool_t *pool = NULL; /* Becomes non-null if mempool used */ 7861d3576fdSSage Weil struct page *page; 7871d3576fdSSage Weil int want; 7885b64640cSYan, Zheng u64 offset = 0, len = 0; 7891d3576fdSSage Weil 7901d3576fdSSage Weil max_pages = max_pages_ever; 7911d3576fdSSage Weil 7921d3576fdSSage Weil get_more_pages: 7931d3576fdSSage Weil first = -1; 7941d3576fdSSage Weil want = min(end - index, 7951d3576fdSSage Weil min((pgoff_t)PAGEVEC_SIZE, 7961d3576fdSSage Weil max_pages - (pgoff_t)locked_pages) - 1) 7971d3576fdSSage Weil + 1; 7981d3576fdSSage Weil pvec_pages = pagevec_lookup_tag(&pvec, mapping, &index, 7991d3576fdSSage Weil PAGECACHE_TAG_DIRTY, 8001d3576fdSSage Weil want); 8011d3576fdSSage Weil dout("pagevec_lookup_tag got %d\n", pvec_pages); 8021d3576fdSSage Weil if (!pvec_pages && !locked_pages) 8031d3576fdSSage Weil break; 8041d3576fdSSage Weil for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { 8051d3576fdSSage Weil page = pvec.pages[i]; 8061d3576fdSSage Weil dout("? %p idx %lu\n", page, page->index); 8071d3576fdSSage Weil if (locked_pages == 0) 8081d3576fdSSage Weil lock_page(page); /* first page */ 8091d3576fdSSage Weil else if (!trylock_page(page)) 8101d3576fdSSage Weil break; 8111d3576fdSSage Weil 8121d3576fdSSage Weil /* only dirty pages, or our accounting breaks */ 8131d3576fdSSage Weil if (unlikely(!PageDirty(page)) || 8141d3576fdSSage Weil unlikely(page->mapping != mapping)) { 8151d3576fdSSage Weil dout("!dirty or !mapping %p\n", page); 8161d3576fdSSage Weil unlock_page(page); 8171d3576fdSSage Weil break; 8181d3576fdSSage Weil } 8191d3576fdSSage Weil if (!wbc->range_cyclic && page->index > end) { 8201d3576fdSSage Weil dout("end of range %p\n", page); 8211d3576fdSSage Weil done = 1; 8221d3576fdSSage Weil unlock_page(page); 8231d3576fdSSage Weil break; 8241d3576fdSSage Weil } 8255b64640cSYan, Zheng if (strip_unit_end && (page->index > strip_unit_end)) { 8265b64640cSYan, Zheng dout("end of strip unit %p\n", page); 8271d3576fdSSage Weil unlock_page(page); 8281d3576fdSSage Weil break; 8291d3576fdSSage Weil } 8301d3576fdSSage Weil if (wbc->sync_mode != WB_SYNC_NONE) { 8311d3576fdSSage Weil dout("waiting on writeback %p\n", page); 8321d3576fdSSage Weil wait_on_page_writeback(page); 8331d3576fdSSage Weil } 834e1966b49SYan, Zheng if (page_offset(page) >= 835e1966b49SYan, Zheng (snap_size == -1 ? i_size : snap_size)) { 836e1966b49SYan, Zheng dout("%p page eof %llu\n", page, 837e1966b49SYan, Zheng (snap_size == -1 ? i_size : snap_size)); 8381d3576fdSSage Weil done = 1; 8391d3576fdSSage Weil unlock_page(page); 8401d3576fdSSage Weil break; 8411d3576fdSSage Weil } 8421d3576fdSSage Weil if (PageWriteback(page)) { 8431d3576fdSSage Weil dout("%p under writeback\n", page); 8441d3576fdSSage Weil unlock_page(page); 8451d3576fdSSage Weil break; 8461d3576fdSSage Weil } 8471d3576fdSSage Weil 8481d3576fdSSage Weil /* only if matching snap context */ 84961600ef8SYan, Zheng pgsnapc = page_snap_context(page); 85080e755feSSage Weil if (pgsnapc->seq > snapc->seq) { 85180e755feSSage Weil dout("page snapc %p %lld > oldest %p %lld\n", 85280e755feSSage Weil pgsnapc, pgsnapc->seq, snapc, snapc->seq); 8531d3576fdSSage Weil unlock_page(page); 8541d3576fdSSage Weil if (!locked_pages) 8551d3576fdSSage Weil continue; /* keep looking for snap */ 8561d3576fdSSage Weil break; 8571d3576fdSSage Weil } 8581d3576fdSSage Weil 8591d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) { 8601d3576fdSSage Weil dout("%p !clear_page_dirty_for_io\n", page); 8611d3576fdSSage Weil unlock_page(page); 8621d3576fdSSage Weil break; 8631d3576fdSSage Weil } 8641d3576fdSSage Weil 865e5975c7cSAlex Elder /* 866e5975c7cSAlex Elder * We have something to write. If this is 867e5975c7cSAlex Elder * the first locked page this time through, 8685b64640cSYan, Zheng * calculate max possinle write size and 8695b64640cSYan, Zheng * allocate a page array 870e5975c7cSAlex Elder */ 8711d3576fdSSage Weil if (locked_pages == 0) { 8725b64640cSYan, Zheng u64 objnum; 8735b64640cSYan, Zheng u64 objoff; 8745b64640cSYan, Zheng 8751d3576fdSSage Weil /* prepare async write request */ 8766285bc23SAlex Elder offset = (u64)page_offset(page); 8771d3576fdSSage Weil len = wsize; 8785b64640cSYan, Zheng 8795b64640cSYan, Zheng rc = ceph_calc_file_object_mapping(&ci->i_layout, 8805b64640cSYan, Zheng offset, len, 8815b64640cSYan, Zheng &objnum, &objoff, 8825b64640cSYan, Zheng &len); 8835b64640cSYan, Zheng if (rc < 0) { 8848c71897bSHenry C Chang unlock_page(page); 8858c71897bSHenry C Chang break; 8868c71897bSHenry C Chang } 8878c71897bSHenry C Chang 8885b64640cSYan, Zheng num_ops = 1 + do_sync; 8895b64640cSYan, Zheng strip_unit_end = page->index + 8905b64640cSYan, Zheng ((len - 1) >> PAGE_CACHE_SHIFT); 891715e4cd4SYan, Zheng 8925b64640cSYan, Zheng BUG_ON(pages); 89388486957SAlex Elder max_pages = calc_pages_for(0, (u64)len); 894fc2744aaSYan, Zheng pages = kmalloc(max_pages * sizeof (*pages), 895fc2744aaSYan, Zheng GFP_NOFS); 89688486957SAlex Elder if (!pages) { 89788486957SAlex Elder pool = fsc->wb_pagevec_pool; 89888486957SAlex Elder pages = mempool_alloc(pool, GFP_NOFS); 899e5975c7cSAlex Elder BUG_ON(!pages); 90088486957SAlex Elder } 9015b64640cSYan, Zheng 9025b64640cSYan, Zheng len = 0; 9035b64640cSYan, Zheng } else if (page->index != 9045b64640cSYan, Zheng (offset + len) >> PAGE_CACHE_SHIFT) { 9055b64640cSYan, Zheng if (num_ops >= (pool ? CEPH_OSD_SLAB_OPS : 9065b64640cSYan, Zheng CEPH_OSD_MAX_OPS)) { 9075b64640cSYan, Zheng redirty_page_for_writepage(wbc, page); 9085b64640cSYan, Zheng unlock_page(page); 9095b64640cSYan, Zheng break; 9105b64640cSYan, Zheng } 9115b64640cSYan, Zheng 9125b64640cSYan, Zheng num_ops++; 9135b64640cSYan, Zheng offset = (u64)page_offset(page); 9145b64640cSYan, Zheng len = 0; 9151d3576fdSSage Weil } 9161d3576fdSSage Weil 9171d3576fdSSage Weil /* note position of first page in pvec */ 9181d3576fdSSage Weil if (first < 0) 9191d3576fdSSage Weil first = i; 9201d3576fdSSage Weil dout("%p will write page %p idx %lu\n", 9211d3576fdSSage Weil inode, page, page->index); 9222baba250SYehuda Sadeh 9235b64640cSYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 9245b64640cSYan, Zheng CONGESTION_ON_THRESH( 9253d14c5d2SYehuda Sadeh fsc->mount_options->congestion_kb)) { 9263d14c5d2SYehuda Sadeh set_bdi_congested(&fsc->backing_dev_info, 927213c99eeSSage Weil BLK_RW_ASYNC); 9282baba250SYehuda Sadeh } 9292baba250SYehuda Sadeh 930e5975c7cSAlex Elder pages[locked_pages] = page; 9311d3576fdSSage Weil locked_pages++; 9325b64640cSYan, Zheng len += PAGE_CACHE_SIZE; 9331d3576fdSSage Weil } 9341d3576fdSSage Weil 9351d3576fdSSage Weil /* did we get anything? */ 9361d3576fdSSage Weil if (!locked_pages) 9371d3576fdSSage Weil goto release_pvec_pages; 9381d3576fdSSage Weil if (i) { 9391d3576fdSSage Weil int j; 9401d3576fdSSage Weil BUG_ON(!locked_pages || first < 0); 9411d3576fdSSage Weil 9421d3576fdSSage Weil if (pvec_pages && i == pvec_pages && 9431d3576fdSSage Weil locked_pages < max_pages) { 9441d3576fdSSage Weil dout("reached end pvec, trying for more\n"); 9451d3576fdSSage Weil pagevec_reinit(&pvec); 9461d3576fdSSage Weil goto get_more_pages; 9471d3576fdSSage Weil } 9481d3576fdSSage Weil 9491d3576fdSSage Weil /* shift unused pages over in the pvec... we 9501d3576fdSSage Weil * will need to release them below. */ 9511d3576fdSSage Weil for (j = i; j < pvec_pages; j++) { 9525b64640cSYan, Zheng dout(" pvec leftover page %p\n", pvec.pages[j]); 9531d3576fdSSage Weil pvec.pages[j-i+first] = pvec.pages[j]; 9541d3576fdSSage Weil } 9551d3576fdSSage Weil pvec.nr -= i-first; 9561d3576fdSSage Weil } 9571d3576fdSSage Weil 9585b64640cSYan, Zheng new_request: 959e5975c7cSAlex Elder offset = page_offset(pages[0]); 9605b64640cSYan, Zheng len = wsize; 9615b64640cSYan, Zheng 9625b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 9635b64640cSYan, Zheng &ci->i_layout, vino, 9645b64640cSYan, Zheng offset, &len, 0, num_ops, 9655b64640cSYan, Zheng CEPH_OSD_OP_WRITE, 9665b64640cSYan, Zheng CEPH_OSD_FLAG_WRITE | 9675b64640cSYan, Zheng CEPH_OSD_FLAG_ONDISK, 9685b64640cSYan, Zheng snapc, truncate_seq, 9695b64640cSYan, Zheng truncate_size, false); 9705b64640cSYan, Zheng if (IS_ERR(req)) { 9715b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 9725b64640cSYan, Zheng &ci->i_layout, vino, 9735b64640cSYan, Zheng offset, &len, 0, 9745b64640cSYan, Zheng min(num_ops, 9755b64640cSYan, Zheng CEPH_OSD_SLAB_OPS), 9765b64640cSYan, Zheng CEPH_OSD_OP_WRITE, 9775b64640cSYan, Zheng CEPH_OSD_FLAG_WRITE | 9785b64640cSYan, Zheng CEPH_OSD_FLAG_ONDISK, 9795b64640cSYan, Zheng snapc, truncate_seq, 9805b64640cSYan, Zheng truncate_size, true); 9815b64640cSYan, Zheng BUG_ON(IS_ERR(req)); 9825b64640cSYan, Zheng } 9835b64640cSYan, Zheng BUG_ON(len < page_offset(pages[locked_pages - 1]) + 9845b64640cSYan, Zheng PAGE_CACHE_SIZE - offset); 9855b64640cSYan, Zheng 9865b64640cSYan, Zheng req->r_callback = writepages_finish; 9875b64640cSYan, Zheng req->r_inode = inode; 9885b64640cSYan, Zheng 9895b64640cSYan, Zheng /* Format the osd request message and submit the write */ 9905b64640cSYan, Zheng len = 0; 9915b64640cSYan, Zheng data_pages = pages; 9925b64640cSYan, Zheng op_idx = 0; 9935b64640cSYan, Zheng for (i = 0; i < locked_pages; i++) { 9945b64640cSYan, Zheng u64 cur_offset = page_offset(pages[i]); 9955b64640cSYan, Zheng if (offset + len != cur_offset) { 9965b64640cSYan, Zheng if (op_idx + do_sync + 1 == req->r_num_ops) 9975b64640cSYan, Zheng break; 9985b64640cSYan, Zheng osd_req_op_extent_dup_last(req, op_idx, 9995b64640cSYan, Zheng cur_offset - offset); 10005b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", 10015b64640cSYan, Zheng offset, len); 10025b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, 10035b64640cSYan, Zheng data_pages, len, 0, 10045b64640cSYan, Zheng !!pool, false); 10055b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 10065b64640cSYan, Zheng 10075b64640cSYan, Zheng len = 0; 10085b64640cSYan, Zheng offset = cur_offset; 10095b64640cSYan, Zheng data_pages = pages + i; 10105b64640cSYan, Zheng op_idx++; 10115b64640cSYan, Zheng } 10125b64640cSYan, Zheng 10135b64640cSYan, Zheng set_page_writeback(pages[i]); 10145b64640cSYan, Zheng len += PAGE_CACHE_SIZE; 10155b64640cSYan, Zheng } 10165b64640cSYan, Zheng 10175b64640cSYan, Zheng if (snap_size != -1) { 10185b64640cSYan, Zheng len = min(len, snap_size - offset); 10195b64640cSYan, Zheng } else if (i == locked_pages) { 1020e1966b49SYan, Zheng /* writepages_finish() clears writeback pages 1021e1966b49SYan, Zheng * according to the data length, so make sure 1022e1966b49SYan, Zheng * data length covers all locked pages */ 10235b64640cSYan, Zheng u64 min_len = len + 1 - PAGE_CACHE_SIZE; 10245b64640cSYan, Zheng len = min(len, (u64)i_size_read(inode) - offset); 10255b64640cSYan, Zheng len = max(len, min_len); 1026e1966b49SYan, Zheng } 10275b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", offset, len); 10281d3576fdSSage Weil 10295b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, data_pages, len, 10305b64640cSYan, Zheng 0, !!pool, false); 10315b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 1032e5975c7cSAlex Elder 10335b64640cSYan, Zheng if (do_sync) { 10345b64640cSYan, Zheng op_idx++; 10355b64640cSYan, Zheng osd_req_op_init(req, op_idx, CEPH_OSD_OP_STARTSYNC, 0); 10365b64640cSYan, Zheng } 10375b64640cSYan, Zheng BUG_ON(op_idx + 1 != req->r_num_ops); 10385b64640cSYan, Zheng 1039e5975c7cSAlex Elder pool = NULL; 10405b64640cSYan, Zheng if (i < locked_pages) { 10415b64640cSYan, Zheng BUG_ON(num_ops <= req->r_num_ops); 10425b64640cSYan, Zheng num_ops -= req->r_num_ops; 10435b64640cSYan, Zheng num_ops += do_sync; 10445b64640cSYan, Zheng locked_pages -= i; 1045e5975c7cSAlex Elder 10465b64640cSYan, Zheng /* allocate new pages array for next request */ 10475b64640cSYan, Zheng data_pages = pages; 10485b64640cSYan, Zheng pages = kmalloc(locked_pages * sizeof (*pages), 10495b64640cSYan, Zheng GFP_NOFS); 10505b64640cSYan, Zheng if (!pages) { 10515b64640cSYan, Zheng pool = fsc->wb_pagevec_pool; 10525b64640cSYan, Zheng pages = mempool_alloc(pool, GFP_NOFS); 10535b64640cSYan, Zheng BUG_ON(!pages); 10545b64640cSYan, Zheng } 10555b64640cSYan, Zheng memcpy(pages, data_pages + i, 10565b64640cSYan, Zheng locked_pages * sizeof(*pages)); 10575b64640cSYan, Zheng memset(data_pages + i, 0, 10585b64640cSYan, Zheng locked_pages * sizeof(*pages)); 10595b64640cSYan, Zheng } else { 10605b64640cSYan, Zheng BUG_ON(num_ops != req->r_num_ops); 10615b64640cSYan, Zheng index = pages[i - 1]->index + 1; 10625b64640cSYan, Zheng /* request message now owns the pages array */ 10635b64640cSYan, Zheng pages = NULL; 10645b64640cSYan, Zheng } 1065e5975c7cSAlex Elder 1066e5975c7cSAlex Elder vino = ceph_vino(inode); 106779528734SAlex Elder ceph_osdc_build_request(req, offset, snapc, vino.snap, 106879528734SAlex Elder &inode->i_mtime); 10691d3576fdSSage Weil 10709d6fcb08SSage Weil rc = ceph_osdc_start_request(&fsc->client->osdc, req, true); 10719d6fcb08SSage Weil BUG_ON(rc); 10721d3576fdSSage Weil req = NULL; 10731d3576fdSSage Weil 10745b64640cSYan, Zheng wbc->nr_to_write -= i; 10755b64640cSYan, Zheng if (pages) 10765b64640cSYan, Zheng goto new_request; 10775b64640cSYan, Zheng 10781d3576fdSSage Weil if (wbc->nr_to_write <= 0) 10791d3576fdSSage Weil done = 1; 10801d3576fdSSage Weil 10811d3576fdSSage Weil release_pvec_pages: 10821d3576fdSSage Weil dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, 10831d3576fdSSage Weil pvec.nr ? pvec.pages[0] : NULL); 10841d3576fdSSage Weil pagevec_release(&pvec); 10851d3576fdSSage Weil 10861d3576fdSSage Weil if (locked_pages && !done) 10871d3576fdSSage Weil goto retry; 10881d3576fdSSage Weil } 10891d3576fdSSage Weil 10901d3576fdSSage Weil if (should_loop && !done) { 10911d3576fdSSage Weil /* more to do; loop back to beginning of file */ 10921d3576fdSSage Weil dout("writepages looping back to beginning of file\n"); 10931d3576fdSSage Weil should_loop = 0; 10941d3576fdSSage Weil index = 0; 10951d3576fdSSage Weil goto retry; 10961d3576fdSSage Weil } 10971d3576fdSSage Weil 10981d3576fdSSage Weil if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 10991d3576fdSSage Weil mapping->writeback_index = index; 11001d3576fdSSage Weil 11011d3576fdSSage Weil out: 11021d3576fdSSage Weil if (req) 11031d3576fdSSage Weil ceph_osdc_put_request(req); 11041d3576fdSSage Weil ceph_put_snap_context(snapc); 11051d3576fdSSage Weil dout("writepages done, rc = %d\n", rc); 11061d3576fdSSage Weil return rc; 11071d3576fdSSage Weil } 11081d3576fdSSage Weil 11091d3576fdSSage Weil 11101d3576fdSSage Weil 11111d3576fdSSage Weil /* 11121d3576fdSSage Weil * See if a given @snapc is either writeable, or already written. 11131d3576fdSSage Weil */ 11141d3576fdSSage Weil static int context_is_writeable_or_written(struct inode *inode, 11151d3576fdSSage Weil struct ceph_snap_context *snapc) 11161d3576fdSSage Weil { 11171d3576fdSSage Weil struct ceph_snap_context *oldest = get_oldest_context(inode, NULL); 11186298a337SSage Weil int ret = !oldest || snapc->seq <= oldest->seq; 11196298a337SSage Weil 11206298a337SSage Weil ceph_put_snap_context(oldest); 11216298a337SSage Weil return ret; 11221d3576fdSSage Weil } 11231d3576fdSSage Weil 11241d3576fdSSage Weil /* 11251d3576fdSSage Weil * We are only allowed to write into/dirty the page if the page is 11261d3576fdSSage Weil * clean, or already dirty within the same snap context. 11278f883c24SSage Weil * 11288f883c24SSage Weil * called with page locked. 11298f883c24SSage Weil * return success with page locked, 11308f883c24SSage Weil * or any failure (incl -EAGAIN) with page unlocked. 11311d3576fdSSage Weil */ 11324af6b225SYehuda Sadeh static int ceph_update_writeable_page(struct file *file, 11334af6b225SYehuda Sadeh loff_t pos, unsigned len, 11344af6b225SYehuda Sadeh struct page *page) 11351d3576fdSSage Weil { 1136496ad9aaSAl Viro struct inode *inode = file_inode(file); 11371d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 11381d3576fdSSage Weil loff_t page_off = pos & PAGE_CACHE_MASK; 11391d3576fdSSage Weil int pos_in_page = pos & ~PAGE_CACHE_MASK; 11401d3576fdSSage Weil int end_in_page = pos_in_page + len; 11411d3576fdSSage Weil loff_t i_size; 11421d3576fdSSage Weil int r; 114380e755feSSage Weil struct ceph_snap_context *snapc, *oldest; 11441d3576fdSSage Weil 11451d3576fdSSage Weil retry_locked: 11461d3576fdSSage Weil /* writepages currently holds page lock, but if we change that later, */ 11471d3576fdSSage Weil wait_on_page_writeback(page); 11481d3576fdSSage Weil 114961600ef8SYan, Zheng snapc = page_snap_context(page); 115080e755feSSage Weil if (snapc && snapc != ci->i_head_snapc) { 11511d3576fdSSage Weil /* 11521d3576fdSSage Weil * this page is already dirty in another (older) snap 11531d3576fdSSage Weil * context! is it writeable now? 11541d3576fdSSage Weil */ 115580e755feSSage Weil oldest = get_oldest_context(inode, NULL); 11561d3576fdSSage Weil 115780e755feSSage Weil if (snapc->seq > oldest->seq) { 11586298a337SSage Weil ceph_put_snap_context(oldest); 11591d3576fdSSage Weil dout(" page %p snapc %p not current or oldest\n", 11606298a337SSage Weil page, snapc); 11611d3576fdSSage Weil /* 11621d3576fdSSage Weil * queue for writeback, and wait for snapc to 11631d3576fdSSage Weil * be writeable or written 11641d3576fdSSage Weil */ 11656298a337SSage Weil snapc = ceph_get_snap_context(snapc); 11661d3576fdSSage Weil unlock_page(page); 11673c6f6b79SSage Weil ceph_queue_writeback(inode); 11688f883c24SSage Weil r = wait_event_interruptible(ci->i_cap_wq, 11691d3576fdSSage Weil context_is_writeable_or_written(inode, snapc)); 11701d3576fdSSage Weil ceph_put_snap_context(snapc); 11718f883c24SSage Weil if (r == -ERESTARTSYS) 11728f883c24SSage Weil return r; 11734af6b225SYehuda Sadeh return -EAGAIN; 11741d3576fdSSage Weil } 11756298a337SSage Weil ceph_put_snap_context(oldest); 11761d3576fdSSage Weil 11771d3576fdSSage Weil /* yay, writeable, do it now (without dropping page lock) */ 11781d3576fdSSage Weil dout(" page %p snapc %p not current, but oldest\n", 11791d3576fdSSage Weil page, snapc); 11801d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) 11811d3576fdSSage Weil goto retry_locked; 11821d3576fdSSage Weil r = writepage_nounlock(page, NULL); 11831d3576fdSSage Weil if (r < 0) 11841d3576fdSSage Weil goto fail_nosnap; 11851d3576fdSSage Weil goto retry_locked; 11861d3576fdSSage Weil } 11871d3576fdSSage Weil 11881d3576fdSSage Weil if (PageUptodate(page)) { 11891d3576fdSSage Weil dout(" page %p already uptodate\n", page); 11901d3576fdSSage Weil return 0; 11911d3576fdSSage Weil } 11921d3576fdSSage Weil 11931d3576fdSSage Weil /* full page? */ 11941d3576fdSSage Weil if (pos_in_page == 0 && len == PAGE_CACHE_SIZE) 11951d3576fdSSage Weil return 0; 11961d3576fdSSage Weil 11971d3576fdSSage Weil /* past end of file? */ 119899c88e69SYan, Zheng i_size = i_size_read(inode); 11991d3576fdSSage Weil 12001d3576fdSSage Weil if (page_off >= i_size || 12011d3576fdSSage Weil (pos_in_page == 0 && (pos+len) >= i_size && 12021d3576fdSSage Weil end_in_page - pos_in_page != PAGE_CACHE_SIZE)) { 12031d3576fdSSage Weil dout(" zeroing %p 0 - %d and %d - %d\n", 12041d3576fdSSage Weil page, pos_in_page, end_in_page, (int)PAGE_CACHE_SIZE); 12051d3576fdSSage Weil zero_user_segments(page, 12061d3576fdSSage Weil 0, pos_in_page, 12071d3576fdSSage Weil end_in_page, PAGE_CACHE_SIZE); 12081d3576fdSSage Weil return 0; 12091d3576fdSSage Weil } 12101d3576fdSSage Weil 12111d3576fdSSage Weil /* we need to read it. */ 12121d3576fdSSage Weil r = readpage_nounlock(file, page); 12131d3576fdSSage Weil if (r < 0) 12141d3576fdSSage Weil goto fail_nosnap; 12151d3576fdSSage Weil goto retry_locked; 12161d3576fdSSage Weil fail_nosnap: 12171d3576fdSSage Weil unlock_page(page); 12181d3576fdSSage Weil return r; 12191d3576fdSSage Weil } 12201d3576fdSSage Weil 12211d3576fdSSage Weil /* 12224af6b225SYehuda Sadeh * We are only allowed to write into/dirty the page if the page is 12234af6b225SYehuda Sadeh * clean, or already dirty within the same snap context. 12244af6b225SYehuda Sadeh */ 12254af6b225SYehuda Sadeh static int ceph_write_begin(struct file *file, struct address_space *mapping, 12264af6b225SYehuda Sadeh loff_t pos, unsigned len, unsigned flags, 12274af6b225SYehuda Sadeh struct page **pagep, void **fsdata) 12284af6b225SYehuda Sadeh { 1229496ad9aaSAl Viro struct inode *inode = file_inode(file); 12304af6b225SYehuda Sadeh struct page *page; 12314af6b225SYehuda Sadeh pgoff_t index = pos >> PAGE_CACHE_SHIFT; 12327971bd92SSage Weil int r; 12334af6b225SYehuda Sadeh 12344af6b225SYehuda Sadeh do { 12354af6b225SYehuda Sadeh /* get a page */ 12364af6b225SYehuda Sadeh page = grab_cache_page_write_begin(mapping, index, 0); 12377971bd92SSage Weil if (!page) 12387971bd92SSage Weil return -ENOMEM; 12394af6b225SYehuda Sadeh 12404af6b225SYehuda Sadeh dout("write_begin file %p inode %p page %p %d~%d\n", file, 12414af6b225SYehuda Sadeh inode, page, (int)pos, (int)len); 12424af6b225SYehuda Sadeh 12434af6b225SYehuda Sadeh r = ceph_update_writeable_page(file, pos, len, page); 1244c1d00b2dSTaesoo Kim if (r < 0) 1245c1d00b2dSTaesoo Kim page_cache_release(page); 1246c1d00b2dSTaesoo Kim else 1247c1d00b2dSTaesoo Kim *pagep = page; 12484af6b225SYehuda Sadeh } while (r == -EAGAIN); 12494af6b225SYehuda Sadeh 12504af6b225SYehuda Sadeh return r; 12514af6b225SYehuda Sadeh } 12524af6b225SYehuda Sadeh 12534af6b225SYehuda Sadeh /* 12541d3576fdSSage Weil * we don't do anything in here that simple_write_end doesn't do 12555dda377cSYan, Zheng * except adjust dirty page accounting 12561d3576fdSSage Weil */ 12571d3576fdSSage Weil static int ceph_write_end(struct file *file, struct address_space *mapping, 12581d3576fdSSage Weil loff_t pos, unsigned len, unsigned copied, 12591d3576fdSSage Weil struct page *page, void *fsdata) 12601d3576fdSSage Weil { 1261496ad9aaSAl Viro struct inode *inode = file_inode(file); 12621d3576fdSSage Weil unsigned from = pos & (PAGE_CACHE_SIZE - 1); 12631d3576fdSSage Weil int check_cap = 0; 12641d3576fdSSage Weil 12651d3576fdSSage Weil dout("write_end file %p inode %p page %p %d~%d (%d)\n", file, 12661d3576fdSSage Weil inode, page, (int)pos, (int)copied, (int)len); 12671d3576fdSSage Weil 12681d3576fdSSage Weil /* zero the stale part of the page if we did a short copy */ 12691d3576fdSSage Weil if (copied < len) 12701d3576fdSSage Weil zero_user_segment(page, from+copied, len); 12711d3576fdSSage Weil 12721d3576fdSSage Weil /* did file size increase? */ 127399c88e69SYan, Zheng if (pos+copied > i_size_read(inode)) 12741d3576fdSSage Weil check_cap = ceph_inode_set_size(inode, pos+copied); 12751d3576fdSSage Weil 12761d3576fdSSage Weil if (!PageUptodate(page)) 12771d3576fdSSage Weil SetPageUptodate(page); 12781d3576fdSSage Weil 12791d3576fdSSage Weil set_page_dirty(page); 12801d3576fdSSage Weil 12811d3576fdSSage Weil unlock_page(page); 12821d3576fdSSage Weil page_cache_release(page); 12831d3576fdSSage Weil 12841d3576fdSSage Weil if (check_cap) 12851d3576fdSSage Weil ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY, NULL); 12861d3576fdSSage Weil 12871d3576fdSSage Weil return copied; 12881d3576fdSSage Weil } 12891d3576fdSSage Weil 12901d3576fdSSage Weil /* 12911d3576fdSSage Weil * we set .direct_IO to indicate direct io is supported, but since we 12921d3576fdSSage Weil * intercept O_DIRECT reads and writes early, this function should 12931d3576fdSSage Weil * never get called. 12941d3576fdSSage Weil */ 129522c6186eSOmar Sandoval static ssize_t ceph_direct_io(struct kiocb *iocb, struct iov_iter *iter, 1296d8d3d94bSAl Viro loff_t pos) 12971d3576fdSSage Weil { 12981d3576fdSSage Weil WARN_ON(1); 12991d3576fdSSage Weil return -EINVAL; 13001d3576fdSSage Weil } 13011d3576fdSSage Weil 13021d3576fdSSage Weil const struct address_space_operations ceph_aops = { 13031d3576fdSSage Weil .readpage = ceph_readpage, 13041d3576fdSSage Weil .readpages = ceph_readpages, 13051d3576fdSSage Weil .writepage = ceph_writepage, 13061d3576fdSSage Weil .writepages = ceph_writepages_start, 13071d3576fdSSage Weil .write_begin = ceph_write_begin, 13081d3576fdSSage Weil .write_end = ceph_write_end, 13091d3576fdSSage Weil .set_page_dirty = ceph_set_page_dirty, 13101d3576fdSSage Weil .invalidatepage = ceph_invalidatepage, 13111d3576fdSSage Weil .releasepage = ceph_releasepage, 13121d3576fdSSage Weil .direct_IO = ceph_direct_io, 13131d3576fdSSage Weil }; 13141d3576fdSSage Weil 13151d3576fdSSage Weil 13161d3576fdSSage Weil /* 13171d3576fdSSage Weil * vm ops 13181d3576fdSSage Weil */ 131961f68816SYan, Zheng static int ceph_filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 132061f68816SYan, Zheng { 132161f68816SYan, Zheng struct inode *inode = file_inode(vma->vm_file); 132261f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 132361f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 13243738daa6SYan, Zheng struct page *pinned_page = NULL; 132561f68816SYan, Zheng loff_t off = vmf->pgoff << PAGE_CACHE_SHIFT; 132661f68816SYan, Zheng int want, got, ret; 132761f68816SYan, Zheng 132861f68816SYan, Zheng dout("filemap_fault %p %llx.%llx %llu~%zd trying to get caps\n", 132937b52fe6SIlya Dryomov inode, ceph_vinop(inode), off, (size_t)PAGE_CACHE_SIZE); 133061f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 133161f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; 133261f68816SYan, Zheng else 133361f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE; 133461f68816SYan, Zheng while (1) { 133561f68816SYan, Zheng got = 0; 133683701246SYan, Zheng ret = ceph_get_caps(ci, CEPH_CAP_FILE_RD, want, 133783701246SYan, Zheng -1, &got, &pinned_page); 133861f68816SYan, Zheng if (ret == 0) 133961f68816SYan, Zheng break; 134061f68816SYan, Zheng if (ret != -ERESTARTSYS) { 134161f68816SYan, Zheng WARN_ON(1); 134261f68816SYan, Zheng return VM_FAULT_SIGBUS; 134361f68816SYan, Zheng } 134461f68816SYan, Zheng } 134561f68816SYan, Zheng dout("filemap_fault %p %llu~%zd got cap refs on %s\n", 134637b52fe6SIlya Dryomov inode, off, (size_t)PAGE_CACHE_SIZE, ceph_cap_string(got)); 134761f68816SYan, Zheng 134883701246SYan, Zheng if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || 134983701246SYan, Zheng ci->i_inline_version == CEPH_INLINE_NONE) 135061f68816SYan, Zheng ret = filemap_fault(vma, vmf); 135183701246SYan, Zheng else 135283701246SYan, Zheng ret = -EAGAIN; 135361f68816SYan, Zheng 135461f68816SYan, Zheng dout("filemap_fault %p %llu~%zd dropping cap refs on %s ret %d\n", 135537b52fe6SIlya Dryomov inode, off, (size_t)PAGE_CACHE_SIZE, ceph_cap_string(got), ret); 13563738daa6SYan, Zheng if (pinned_page) 13573738daa6SYan, Zheng page_cache_release(pinned_page); 135861f68816SYan, Zheng ceph_put_cap_refs(ci, got); 135961f68816SYan, Zheng 136083701246SYan, Zheng if (ret != -EAGAIN) 136183701246SYan, Zheng return ret; 136283701246SYan, Zheng 136383701246SYan, Zheng /* read inline data */ 136483701246SYan, Zheng if (off >= PAGE_CACHE_SIZE) { 136583701246SYan, Zheng /* does not support inline data > PAGE_SIZE */ 136683701246SYan, Zheng ret = VM_FAULT_SIGBUS; 136783701246SYan, Zheng } else { 136883701246SYan, Zheng int ret1; 136983701246SYan, Zheng struct address_space *mapping = inode->i_mapping; 137083701246SYan, Zheng struct page *page = find_or_create_page(mapping, 0, 1371c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1372c62d2555SMichal Hocko ~__GFP_FS)); 137383701246SYan, Zheng if (!page) { 137483701246SYan, Zheng ret = VM_FAULT_OOM; 137583701246SYan, Zheng goto out; 137683701246SYan, Zheng } 137783701246SYan, Zheng ret1 = __ceph_do_getattr(inode, page, 137883701246SYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 137983701246SYan, Zheng if (ret1 < 0 || off >= i_size_read(inode)) { 138083701246SYan, Zheng unlock_page(page); 138183701246SYan, Zheng page_cache_release(page); 138283701246SYan, Zheng ret = VM_FAULT_SIGBUS; 138383701246SYan, Zheng goto out; 138483701246SYan, Zheng } 138583701246SYan, Zheng if (ret1 < PAGE_CACHE_SIZE) 138683701246SYan, Zheng zero_user_segment(page, ret1, PAGE_CACHE_SIZE); 138783701246SYan, Zheng else 138883701246SYan, Zheng flush_dcache_page(page); 138983701246SYan, Zheng SetPageUptodate(page); 139083701246SYan, Zheng vmf->page = page; 139183701246SYan, Zheng ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; 139283701246SYan, Zheng } 139383701246SYan, Zheng out: 139483701246SYan, Zheng dout("filemap_fault %p %llu~%zd read inline data ret %d\n", 139583701246SYan, Zheng inode, off, (size_t)PAGE_CACHE_SIZE, ret); 139661f68816SYan, Zheng return ret; 139761f68816SYan, Zheng } 13981d3576fdSSage Weil 13991d3576fdSSage Weil /* 14001d3576fdSSage Weil * Reuse write_begin here for simplicity. 14011d3576fdSSage Weil */ 14021d3576fdSSage Weil static int ceph_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 14031d3576fdSSage Weil { 1404496ad9aaSAl Viro struct inode *inode = file_inode(vma->vm_file); 140561f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 140661f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 1407f66fd9f0SYan, Zheng struct ceph_cap_flush *prealloc_cf; 140861f68816SYan, Zheng struct page *page = vmf->page; 14096285bc23SAlex Elder loff_t off = page_offset(page); 141061f68816SYan, Zheng loff_t size = i_size_read(inode); 141161f68816SYan, Zheng size_t len; 141261f68816SYan, Zheng int want, got, ret; 14131d3576fdSSage Weil 1414f66fd9f0SYan, Zheng prealloc_cf = ceph_alloc_cap_flush(); 1415f66fd9f0SYan, Zheng if (!prealloc_cf) 1416f66fd9f0SYan, Zheng return VM_FAULT_SIGBUS; 1417f66fd9f0SYan, Zheng 141828127bddSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 141928127bddSYan, Zheng struct page *locked_page = NULL; 142028127bddSYan, Zheng if (off == 0) { 142128127bddSYan, Zheng lock_page(page); 142228127bddSYan, Zheng locked_page = page; 142328127bddSYan, Zheng } 142428127bddSYan, Zheng ret = ceph_uninline_data(vma->vm_file, locked_page); 142528127bddSYan, Zheng if (locked_page) 142628127bddSYan, Zheng unlock_page(locked_page); 1427f66fd9f0SYan, Zheng if (ret < 0) { 1428f66fd9f0SYan, Zheng ret = VM_FAULT_SIGBUS; 1429f66fd9f0SYan, Zheng goto out_free; 1430f66fd9f0SYan, Zheng } 143128127bddSYan, Zheng } 143228127bddSYan, Zheng 14331d3576fdSSage Weil if (off + PAGE_CACHE_SIZE <= size) 14341d3576fdSSage Weil len = PAGE_CACHE_SIZE; 14351d3576fdSSage Weil else 14361d3576fdSSage Weil len = size & ~PAGE_CACHE_MASK; 14371d3576fdSSage Weil 143861f68816SYan, Zheng dout("page_mkwrite %p %llx.%llx %llu~%zd getting caps i_size %llu\n", 143961f68816SYan, Zheng inode, ceph_vinop(inode), off, len, size); 144061f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 144161f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; 144261f68816SYan, Zheng else 144361f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER; 144461f68816SYan, Zheng while (1) { 144561f68816SYan, Zheng got = 0; 14463738daa6SYan, Zheng ret = ceph_get_caps(ci, CEPH_CAP_FILE_WR, want, off + len, 14473738daa6SYan, Zheng &got, NULL); 144861f68816SYan, Zheng if (ret == 0) 144961f68816SYan, Zheng break; 145061f68816SYan, Zheng if (ret != -ERESTARTSYS) { 145161f68816SYan, Zheng WARN_ON(1); 1452f66fd9f0SYan, Zheng ret = VM_FAULT_SIGBUS; 1453f66fd9f0SYan, Zheng goto out_free; 145461f68816SYan, Zheng } 145561f68816SYan, Zheng } 145661f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd got cap refs on %s\n", 145761f68816SYan, Zheng inode, off, len, ceph_cap_string(got)); 145861f68816SYan, Zheng 145961f68816SYan, Zheng /* Update time before taking page lock */ 146061f68816SYan, Zheng file_update_time(vma->vm_file); 14614af6b225SYehuda Sadeh 14624af6b225SYehuda Sadeh lock_page(page); 14634af6b225SYehuda Sadeh 14644af6b225SYehuda Sadeh ret = VM_FAULT_NOPAGE; 14654af6b225SYehuda Sadeh if ((off > size) || 1466f9cac5acSYan, Zheng (page->mapping != inode->i_mapping)) { 1467f9cac5acSYan, Zheng unlock_page(page); 14684af6b225SYehuda Sadeh goto out; 1469f9cac5acSYan, Zheng } 14704af6b225SYehuda Sadeh 14714af6b225SYehuda Sadeh ret = ceph_update_writeable_page(vma->vm_file, off, len, page); 1472f9cac5acSYan, Zheng if (ret >= 0) { 14734af6b225SYehuda Sadeh /* success. we'll keep the page locked. */ 14741d3576fdSSage Weil set_page_dirty(page); 14751d3576fdSSage Weil ret = VM_FAULT_LOCKED; 14761d3576fdSSage Weil } else { 14774af6b225SYehuda Sadeh if (ret == -ENOMEM) 14784af6b225SYehuda Sadeh ret = VM_FAULT_OOM; 14794af6b225SYehuda Sadeh else 14801d3576fdSSage Weil ret = VM_FAULT_SIGBUS; 14811d3576fdSSage Weil } 14824af6b225SYehuda Sadeh out: 148328127bddSYan, Zheng if (ret == VM_FAULT_LOCKED || 148428127bddSYan, Zheng ci->i_inline_version != CEPH_INLINE_NONE) { 148561f68816SYan, Zheng int dirty; 148661f68816SYan, Zheng spin_lock(&ci->i_ceph_lock); 148728127bddSYan, Zheng ci->i_inline_version = CEPH_INLINE_NONE; 1488f66fd9f0SYan, Zheng dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, 1489f66fd9f0SYan, Zheng &prealloc_cf); 149061f68816SYan, Zheng spin_unlock(&ci->i_ceph_lock); 149161f68816SYan, Zheng if (dirty) 149261f68816SYan, Zheng __mark_inode_dirty(inode, dirty); 149361f68816SYan, Zheng } 149461f68816SYan, Zheng 149561f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd dropping cap refs on %s ret %d\n", 149661f68816SYan, Zheng inode, off, len, ceph_cap_string(got), ret); 149761f68816SYan, Zheng ceph_put_cap_refs(ci, got); 1498f66fd9f0SYan, Zheng out_free: 1499f66fd9f0SYan, Zheng ceph_free_cap_flush(prealloc_cf); 150061f68816SYan, Zheng 15011d3576fdSSage Weil return ret; 15021d3576fdSSage Weil } 15031d3576fdSSage Weil 150431c542a1SYan, Zheng void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, 150531c542a1SYan, Zheng char *data, size_t len) 150631c542a1SYan, Zheng { 150731c542a1SYan, Zheng struct address_space *mapping = inode->i_mapping; 150831c542a1SYan, Zheng struct page *page; 150931c542a1SYan, Zheng 151031c542a1SYan, Zheng if (locked_page) { 151131c542a1SYan, Zheng page = locked_page; 151231c542a1SYan, Zheng } else { 151331c542a1SYan, Zheng if (i_size_read(inode) == 0) 151431c542a1SYan, Zheng return; 151531c542a1SYan, Zheng page = find_or_create_page(mapping, 0, 1516c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1517c62d2555SMichal Hocko ~__GFP_FS)); 151831c542a1SYan, Zheng if (!page) 151931c542a1SYan, Zheng return; 152031c542a1SYan, Zheng if (PageUptodate(page)) { 152131c542a1SYan, Zheng unlock_page(page); 152231c542a1SYan, Zheng page_cache_release(page); 152331c542a1SYan, Zheng return; 152431c542a1SYan, Zheng } 152531c542a1SYan, Zheng } 152631c542a1SYan, Zheng 15270668ff52SIlya Dryomov dout("fill_inline_data %p %llx.%llx len %zu locked_page %p\n", 152831c542a1SYan, Zheng inode, ceph_vinop(inode), len, locked_page); 152931c542a1SYan, Zheng 153031c542a1SYan, Zheng if (len > 0) { 153131c542a1SYan, Zheng void *kaddr = kmap_atomic(page); 153231c542a1SYan, Zheng memcpy(kaddr, data, len); 153331c542a1SYan, Zheng kunmap_atomic(kaddr); 153431c542a1SYan, Zheng } 153531c542a1SYan, Zheng 153631c542a1SYan, Zheng if (page != locked_page) { 153731c542a1SYan, Zheng if (len < PAGE_CACHE_SIZE) 153831c542a1SYan, Zheng zero_user_segment(page, len, PAGE_CACHE_SIZE); 153931c542a1SYan, Zheng else 154031c542a1SYan, Zheng flush_dcache_page(page); 154131c542a1SYan, Zheng 154231c542a1SYan, Zheng SetPageUptodate(page); 154331c542a1SYan, Zheng unlock_page(page); 154431c542a1SYan, Zheng page_cache_release(page); 154531c542a1SYan, Zheng } 154631c542a1SYan, Zheng } 154731c542a1SYan, Zheng 154828127bddSYan, Zheng int ceph_uninline_data(struct file *filp, struct page *locked_page) 154928127bddSYan, Zheng { 155028127bddSYan, Zheng struct inode *inode = file_inode(filp); 155128127bddSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 155228127bddSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 155328127bddSYan, Zheng struct ceph_osd_request *req; 155428127bddSYan, Zheng struct page *page = NULL; 155528127bddSYan, Zheng u64 len, inline_version; 155628127bddSYan, Zheng int err = 0; 155728127bddSYan, Zheng bool from_pagecache = false; 155828127bddSYan, Zheng 155928127bddSYan, Zheng spin_lock(&ci->i_ceph_lock); 156028127bddSYan, Zheng inline_version = ci->i_inline_version; 156128127bddSYan, Zheng spin_unlock(&ci->i_ceph_lock); 156228127bddSYan, Zheng 156328127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu\n", 156428127bddSYan, Zheng inode, ceph_vinop(inode), inline_version); 156528127bddSYan, Zheng 156628127bddSYan, Zheng if (inline_version == 1 || /* initial version, no data */ 156728127bddSYan, Zheng inline_version == CEPH_INLINE_NONE) 156828127bddSYan, Zheng goto out; 156928127bddSYan, Zheng 157028127bddSYan, Zheng if (locked_page) { 157128127bddSYan, Zheng page = locked_page; 157228127bddSYan, Zheng WARN_ON(!PageUptodate(page)); 157328127bddSYan, Zheng } else if (ceph_caps_issued(ci) & 157428127bddSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) { 157528127bddSYan, Zheng page = find_get_page(inode->i_mapping, 0); 157628127bddSYan, Zheng if (page) { 157728127bddSYan, Zheng if (PageUptodate(page)) { 157828127bddSYan, Zheng from_pagecache = true; 157928127bddSYan, Zheng lock_page(page); 158028127bddSYan, Zheng } else { 158128127bddSYan, Zheng page_cache_release(page); 158228127bddSYan, Zheng page = NULL; 158328127bddSYan, Zheng } 158428127bddSYan, Zheng } 158528127bddSYan, Zheng } 158628127bddSYan, Zheng 158728127bddSYan, Zheng if (page) { 158828127bddSYan, Zheng len = i_size_read(inode); 158928127bddSYan, Zheng if (len > PAGE_CACHE_SIZE) 159028127bddSYan, Zheng len = PAGE_CACHE_SIZE; 159128127bddSYan, Zheng } else { 159228127bddSYan, Zheng page = __page_cache_alloc(GFP_NOFS); 159328127bddSYan, Zheng if (!page) { 159428127bddSYan, Zheng err = -ENOMEM; 159528127bddSYan, Zheng goto out; 159628127bddSYan, Zheng } 159728127bddSYan, Zheng err = __ceph_do_getattr(inode, page, 159828127bddSYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 159928127bddSYan, Zheng if (err < 0) { 160028127bddSYan, Zheng /* no inline data */ 160128127bddSYan, Zheng if (err == -ENODATA) 160228127bddSYan, Zheng err = 0; 160328127bddSYan, Zheng goto out; 160428127bddSYan, Zheng } 160528127bddSYan, Zheng len = err; 160628127bddSYan, Zheng } 160728127bddSYan, Zheng 160828127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 160928127bddSYan, Zheng ceph_vino(inode), 0, &len, 0, 1, 161028127bddSYan, Zheng CEPH_OSD_OP_CREATE, 161128127bddSYan, Zheng CEPH_OSD_FLAG_ONDISK | CEPH_OSD_FLAG_WRITE, 16127b06a826SYan, Zheng ceph_empty_snapc, 0, 0, false); 161328127bddSYan, Zheng if (IS_ERR(req)) { 161428127bddSYan, Zheng err = PTR_ERR(req); 161528127bddSYan, Zheng goto out; 161628127bddSYan, Zheng } 161728127bddSYan, Zheng 161828127bddSYan, Zheng ceph_osdc_build_request(req, 0, NULL, CEPH_NOSNAP, &inode->i_mtime); 161928127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 162028127bddSYan, Zheng if (!err) 162128127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 162228127bddSYan, Zheng ceph_osdc_put_request(req); 162328127bddSYan, Zheng if (err < 0) 162428127bddSYan, Zheng goto out; 162528127bddSYan, Zheng 162628127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 162728127bddSYan, Zheng ceph_vino(inode), 0, &len, 1, 3, 162828127bddSYan, Zheng CEPH_OSD_OP_WRITE, 162928127bddSYan, Zheng CEPH_OSD_FLAG_ONDISK | CEPH_OSD_FLAG_WRITE, 16307b06a826SYan, Zheng ceph_empty_snapc, 163128127bddSYan, Zheng ci->i_truncate_seq, ci->i_truncate_size, 163228127bddSYan, Zheng false); 163328127bddSYan, Zheng if (IS_ERR(req)) { 163428127bddSYan, Zheng err = PTR_ERR(req); 163528127bddSYan, Zheng goto out; 163628127bddSYan, Zheng } 163728127bddSYan, Zheng 163828127bddSYan, Zheng osd_req_op_extent_osd_data_pages(req, 1, &page, len, 0, false, false); 163928127bddSYan, Zheng 1640ec137c10SYan, Zheng { 1641ec137c10SYan, Zheng __le64 xattr_buf = cpu_to_le64(inline_version); 164228127bddSYan, Zheng err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, 1643ec137c10SYan, Zheng "inline_version", &xattr_buf, 1644ec137c10SYan, Zheng sizeof(xattr_buf), 164528127bddSYan, Zheng CEPH_OSD_CMPXATTR_OP_GT, 164628127bddSYan, Zheng CEPH_OSD_CMPXATTR_MODE_U64); 164728127bddSYan, Zheng if (err) 164828127bddSYan, Zheng goto out_put; 1649ec137c10SYan, Zheng } 165028127bddSYan, Zheng 1651ec137c10SYan, Zheng { 1652ec137c10SYan, Zheng char xattr_buf[32]; 1653ec137c10SYan, Zheng int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), 1654ec137c10SYan, Zheng "%llu", inline_version); 165528127bddSYan, Zheng err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, 1656ec137c10SYan, Zheng "inline_version", 1657ec137c10SYan, Zheng xattr_buf, xattr_len, 0, 0); 165828127bddSYan, Zheng if (err) 165928127bddSYan, Zheng goto out_put; 1660ec137c10SYan, Zheng } 166128127bddSYan, Zheng 166228127bddSYan, Zheng ceph_osdc_build_request(req, 0, NULL, CEPH_NOSNAP, &inode->i_mtime); 166328127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 166428127bddSYan, Zheng if (!err) 166528127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 166628127bddSYan, Zheng out_put: 166728127bddSYan, Zheng ceph_osdc_put_request(req); 166828127bddSYan, Zheng if (err == -ECANCELED) 166928127bddSYan, Zheng err = 0; 167028127bddSYan, Zheng out: 167128127bddSYan, Zheng if (page && page != locked_page) { 167228127bddSYan, Zheng if (from_pagecache) { 167328127bddSYan, Zheng unlock_page(page); 167428127bddSYan, Zheng page_cache_release(page); 167528127bddSYan, Zheng } else 167628127bddSYan, Zheng __free_pages(page, 0); 167728127bddSYan, Zheng } 167828127bddSYan, Zheng 167928127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu = %d\n", 168028127bddSYan, Zheng inode, ceph_vinop(inode), inline_version, err); 168128127bddSYan, Zheng return err; 168228127bddSYan, Zheng } 168328127bddSYan, Zheng 16847cbea8dcSKirill A. Shutemov static const struct vm_operations_struct ceph_vmops = { 168561f68816SYan, Zheng .fault = ceph_filemap_fault, 16861d3576fdSSage Weil .page_mkwrite = ceph_page_mkwrite, 16871d3576fdSSage Weil }; 16881d3576fdSSage Weil 16891d3576fdSSage Weil int ceph_mmap(struct file *file, struct vm_area_struct *vma) 16901d3576fdSSage Weil { 16911d3576fdSSage Weil struct address_space *mapping = file->f_mapping; 16921d3576fdSSage Weil 16931d3576fdSSage Weil if (!mapping->a_ops->readpage) 16941d3576fdSSage Weil return -ENOEXEC; 16951d3576fdSSage Weil file_accessed(file); 16961d3576fdSSage Weil vma->vm_ops = &ceph_vmops; 16971d3576fdSSage Weil return 0; 16981d3576fdSSage Weil } 169910183a69SYan, Zheng 170010183a69SYan, Zheng enum { 170110183a69SYan, Zheng POOL_READ = 1, 170210183a69SYan, Zheng POOL_WRITE = 2, 170310183a69SYan, Zheng }; 170410183a69SYan, Zheng 170510183a69SYan, Zheng static int __ceph_pool_perm_get(struct ceph_inode_info *ci, u32 pool) 170610183a69SYan, Zheng { 170710183a69SYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(&ci->vfs_inode); 170810183a69SYan, Zheng struct ceph_mds_client *mdsc = fsc->mdsc; 170910183a69SYan, Zheng struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; 171010183a69SYan, Zheng struct rb_node **p, *parent; 171110183a69SYan, Zheng struct ceph_pool_perm *perm; 171210183a69SYan, Zheng struct page **pages; 171310183a69SYan, Zheng int err = 0, err2 = 0, have = 0; 171410183a69SYan, Zheng 171510183a69SYan, Zheng down_read(&mdsc->pool_perm_rwsem); 171610183a69SYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 171710183a69SYan, Zheng while (*p) { 171810183a69SYan, Zheng perm = rb_entry(*p, struct ceph_pool_perm, node); 171910183a69SYan, Zheng if (pool < perm->pool) 172010183a69SYan, Zheng p = &(*p)->rb_left; 172110183a69SYan, Zheng else if (pool > perm->pool) 172210183a69SYan, Zheng p = &(*p)->rb_right; 172310183a69SYan, Zheng else { 172410183a69SYan, Zheng have = perm->perm; 172510183a69SYan, Zheng break; 172610183a69SYan, Zheng } 172710183a69SYan, Zheng } 172810183a69SYan, Zheng up_read(&mdsc->pool_perm_rwsem); 172910183a69SYan, Zheng if (*p) 173010183a69SYan, Zheng goto out; 173110183a69SYan, Zheng 173210183a69SYan, Zheng dout("__ceph_pool_perm_get pool %u no perm cached\n", pool); 173310183a69SYan, Zheng 173410183a69SYan, Zheng down_write(&mdsc->pool_perm_rwsem); 173510183a69SYan, Zheng parent = NULL; 173610183a69SYan, Zheng while (*p) { 173710183a69SYan, Zheng parent = *p; 173810183a69SYan, Zheng perm = rb_entry(parent, struct ceph_pool_perm, node); 173910183a69SYan, Zheng if (pool < perm->pool) 174010183a69SYan, Zheng p = &(*p)->rb_left; 174110183a69SYan, Zheng else if (pool > perm->pool) 174210183a69SYan, Zheng p = &(*p)->rb_right; 174310183a69SYan, Zheng else { 174410183a69SYan, Zheng have = perm->perm; 174510183a69SYan, Zheng break; 174610183a69SYan, Zheng } 174710183a69SYan, Zheng } 174810183a69SYan, Zheng if (*p) { 174910183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 175010183a69SYan, Zheng goto out; 175110183a69SYan, Zheng } 175210183a69SYan, Zheng 175310183a69SYan, Zheng rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, 17547b06a826SYan, Zheng ceph_empty_snapc, 175510183a69SYan, Zheng 1, false, GFP_NOFS); 175610183a69SYan, Zheng if (!rd_req) { 175710183a69SYan, Zheng err = -ENOMEM; 175810183a69SYan, Zheng goto out_unlock; 175910183a69SYan, Zheng } 176010183a69SYan, Zheng 176110183a69SYan, Zheng rd_req->r_flags = CEPH_OSD_FLAG_READ; 176210183a69SYan, Zheng osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); 176310183a69SYan, Zheng rd_req->r_base_oloc.pool = pool; 176410183a69SYan, Zheng snprintf(rd_req->r_base_oid.name, sizeof(rd_req->r_base_oid.name), 176510183a69SYan, Zheng "%llx.00000000", ci->i_vino.ino); 176610183a69SYan, Zheng rd_req->r_base_oid.name_len = strlen(rd_req->r_base_oid.name); 176710183a69SYan, Zheng 176810183a69SYan, Zheng wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, 17697b06a826SYan, Zheng ceph_empty_snapc, 177010183a69SYan, Zheng 1, false, GFP_NOFS); 177110183a69SYan, Zheng if (!wr_req) { 177210183a69SYan, Zheng err = -ENOMEM; 177310183a69SYan, Zheng goto out_unlock; 177410183a69SYan, Zheng } 177510183a69SYan, Zheng 177610183a69SYan, Zheng wr_req->r_flags = CEPH_OSD_FLAG_WRITE | 177710183a69SYan, Zheng CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK; 177810183a69SYan, Zheng osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); 177910183a69SYan, Zheng wr_req->r_base_oloc.pool = pool; 178010183a69SYan, Zheng wr_req->r_base_oid = rd_req->r_base_oid; 178110183a69SYan, Zheng 178210183a69SYan, Zheng /* one page should be large enough for STAT data */ 178310183a69SYan, Zheng pages = ceph_alloc_page_vector(1, GFP_KERNEL); 178410183a69SYan, Zheng if (IS_ERR(pages)) { 178510183a69SYan, Zheng err = PTR_ERR(pages); 178610183a69SYan, Zheng goto out_unlock; 178710183a69SYan, Zheng } 178810183a69SYan, Zheng 178910183a69SYan, Zheng osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, 179010183a69SYan, Zheng 0, false, true); 179110183a69SYan, Zheng ceph_osdc_build_request(rd_req, 0, NULL, CEPH_NOSNAP, 179210183a69SYan, Zheng &ci->vfs_inode.i_mtime); 179310183a69SYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false); 179410183a69SYan, Zheng 179510183a69SYan, Zheng ceph_osdc_build_request(wr_req, 0, NULL, CEPH_NOSNAP, 179610183a69SYan, Zheng &ci->vfs_inode.i_mtime); 179710183a69SYan, Zheng err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false); 179810183a69SYan, Zheng 179910183a69SYan, Zheng if (!err) 180010183a69SYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); 180110183a69SYan, Zheng if (!err2) 180210183a69SYan, Zheng err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); 180310183a69SYan, Zheng 180410183a69SYan, Zheng if (err >= 0 || err == -ENOENT) 180510183a69SYan, Zheng have |= POOL_READ; 180610183a69SYan, Zheng else if (err != -EPERM) 180710183a69SYan, Zheng goto out_unlock; 180810183a69SYan, Zheng 180910183a69SYan, Zheng if (err2 == 0 || err2 == -EEXIST) 181010183a69SYan, Zheng have |= POOL_WRITE; 181110183a69SYan, Zheng else if (err2 != -EPERM) { 181210183a69SYan, Zheng err = err2; 181310183a69SYan, Zheng goto out_unlock; 181410183a69SYan, Zheng } 181510183a69SYan, Zheng 181610183a69SYan, Zheng perm = kmalloc(sizeof(*perm), GFP_NOFS); 181710183a69SYan, Zheng if (!perm) { 181810183a69SYan, Zheng err = -ENOMEM; 181910183a69SYan, Zheng goto out_unlock; 182010183a69SYan, Zheng } 182110183a69SYan, Zheng 182210183a69SYan, Zheng perm->pool = pool; 182310183a69SYan, Zheng perm->perm = have; 182410183a69SYan, Zheng rb_link_node(&perm->node, parent, p); 182510183a69SYan, Zheng rb_insert_color(&perm->node, &mdsc->pool_perm_tree); 182610183a69SYan, Zheng err = 0; 182710183a69SYan, Zheng out_unlock: 182810183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 182910183a69SYan, Zheng 183010183a69SYan, Zheng if (rd_req) 183110183a69SYan, Zheng ceph_osdc_put_request(rd_req); 183210183a69SYan, Zheng if (wr_req) 183310183a69SYan, Zheng ceph_osdc_put_request(wr_req); 183410183a69SYan, Zheng out: 183510183a69SYan, Zheng if (!err) 183610183a69SYan, Zheng err = have; 183710183a69SYan, Zheng dout("__ceph_pool_perm_get pool %u result = %d\n", pool, err); 183810183a69SYan, Zheng return err; 183910183a69SYan, Zheng } 184010183a69SYan, Zheng 184110183a69SYan, Zheng int ceph_pool_perm_check(struct ceph_inode_info *ci, int need) 184210183a69SYan, Zheng { 184310183a69SYan, Zheng u32 pool; 184410183a69SYan, Zheng int ret, flags; 184510183a69SYan, Zheng 18465ea5c5e0SYan, Zheng /* does not support pool namespace yet */ 18475ea5c5e0SYan, Zheng if (ci->i_pool_ns_len) 18485ea5c5e0SYan, Zheng return -EIO; 18495ea5c5e0SYan, Zheng 185010183a69SYan, Zheng if (ceph_test_mount_opt(ceph_inode_to_client(&ci->vfs_inode), 185110183a69SYan, Zheng NOPOOLPERM)) 185210183a69SYan, Zheng return 0; 185310183a69SYan, Zheng 185410183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 185510183a69SYan, Zheng flags = ci->i_ceph_flags; 185610183a69SYan, Zheng pool = ceph_file_layout_pg_pool(ci->i_layout); 185710183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 185810183a69SYan, Zheng check: 185910183a69SYan, Zheng if (flags & CEPH_I_POOL_PERM) { 186010183a69SYan, Zheng if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { 186110183a69SYan, Zheng dout("ceph_pool_perm_check pool %u no read perm\n", 186210183a69SYan, Zheng pool); 186310183a69SYan, Zheng return -EPERM; 186410183a69SYan, Zheng } 186510183a69SYan, Zheng if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { 186610183a69SYan, Zheng dout("ceph_pool_perm_check pool %u no write perm\n", 186710183a69SYan, Zheng pool); 186810183a69SYan, Zheng return -EPERM; 186910183a69SYan, Zheng } 187010183a69SYan, Zheng return 0; 187110183a69SYan, Zheng } 187210183a69SYan, Zheng 187310183a69SYan, Zheng ret = __ceph_pool_perm_get(ci, pool); 187410183a69SYan, Zheng if (ret < 0) 187510183a69SYan, Zheng return ret; 187610183a69SYan, Zheng 187710183a69SYan, Zheng flags = CEPH_I_POOL_PERM; 187810183a69SYan, Zheng if (ret & POOL_READ) 187910183a69SYan, Zheng flags |= CEPH_I_POOL_RD; 188010183a69SYan, Zheng if (ret & POOL_WRITE) 188110183a69SYan, Zheng flags |= CEPH_I_POOL_WR; 188210183a69SYan, Zheng 188310183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 188410183a69SYan, Zheng if (pool == ceph_file_layout_pg_pool(ci->i_layout)) { 188510183a69SYan, Zheng ci->i_ceph_flags = flags; 188610183a69SYan, Zheng } else { 188710183a69SYan, Zheng pool = ceph_file_layout_pg_pool(ci->i_layout); 188810183a69SYan, Zheng flags = ci->i_ceph_flags; 188910183a69SYan, Zheng } 189010183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 189110183a69SYan, Zheng goto check; 189210183a69SYan, Zheng } 189310183a69SYan, Zheng 189410183a69SYan, Zheng void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) 189510183a69SYan, Zheng { 189610183a69SYan, Zheng struct ceph_pool_perm *perm; 189710183a69SYan, Zheng struct rb_node *n; 189810183a69SYan, Zheng 189910183a69SYan, Zheng while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { 190010183a69SYan, Zheng n = rb_first(&mdsc->pool_perm_tree); 190110183a69SYan, Zheng perm = rb_entry(n, struct ceph_pool_perm, node); 190210183a69SYan, Zheng rb_erase(n, &mdsc->pool_perm_tree); 190310183a69SYan, Zheng kfree(perm); 190410183a69SYan, Zheng } 190510183a69SYan, Zheng } 1906