1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 23d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 31d3576fdSSage Weil 41d3576fdSSage Weil #include <linux/backing-dev.h> 51d3576fdSSage Weil #include <linux/fs.h> 61d3576fdSSage Weil #include <linux/mm.h> 71d3576fdSSage Weil #include <linux/pagemap.h> 81d3576fdSSage Weil #include <linux/writeback.h> /* generic_writepages */ 95a0e3ad6STejun Heo #include <linux/slab.h> 101d3576fdSSage Weil #include <linux/pagevec.h> 111d3576fdSSage Weil #include <linux/task_io_accounting_ops.h> 12f361bf4aSIngo Molnar #include <linux/signal.h> 135c308356SJeff Layton #include <linux/iversion.h> 1497e27aaaSXiubo Li #include <linux/ktime.h> 151d3576fdSSage Weil 161d3576fdSSage Weil #include "super.h" 173d14c5d2SYehuda Sadeh #include "mds_client.h" 1899ccbd22SMilosz Tanski #include "cache.h" 1997e27aaaSXiubo Li #include "metric.h" 203d14c5d2SYehuda Sadeh #include <linux/ceph/osd_client.h> 2108c1ac50SIlya Dryomov #include <linux/ceph/striper.h> 221d3576fdSSage Weil 231d3576fdSSage Weil /* 241d3576fdSSage Weil * Ceph address space ops. 251d3576fdSSage Weil * 261d3576fdSSage Weil * There are a few funny things going on here. 271d3576fdSSage Weil * 281d3576fdSSage Weil * The page->private field is used to reference a struct 291d3576fdSSage Weil * ceph_snap_context for _every_ dirty page. This indicates which 301d3576fdSSage Weil * snapshot the page was logically dirtied in, and thus which snap 311d3576fdSSage Weil * context needs to be associated with the osd write during writeback. 321d3576fdSSage Weil * 331d3576fdSSage Weil * Similarly, struct ceph_inode_info maintains a set of counters to 3425985edcSLucas De Marchi * count dirty pages on the inode. In the absence of snapshots, 351d3576fdSSage Weil * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. 361d3576fdSSage Weil * 371d3576fdSSage Weil * When a snapshot is taken (that is, when the client receives 381d3576fdSSage Weil * notification that a snapshot was taken), each inode with caps and 391d3576fdSSage Weil * with dirty pages (dirty pages implies there is a cap) gets a new 401d3576fdSSage Weil * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending 411d3576fdSSage Weil * order, new snaps go to the tail). The i_wrbuffer_ref_head count is 421d3576fdSSage Weil * moved to capsnap->dirty. (Unless a sync write is currently in 431d3576fdSSage Weil * progress. In that case, the capsnap is said to be "pending", new 441d3576fdSSage Weil * writes cannot start, and the capsnap isn't "finalized" until the 451d3576fdSSage Weil * write completes (or fails) and a final size/mtime for the inode for 461d3576fdSSage Weil * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. 471d3576fdSSage Weil * 481d3576fdSSage Weil * On writeback, we must submit writes to the osd IN SNAP ORDER. So, 491d3576fdSSage Weil * we look for the first capsnap in i_cap_snaps and write out pages in 501d3576fdSSage Weil * that snap context _only_. Then we move on to the next capsnap, 511d3576fdSSage Weil * eventually reaching the "live" or "head" context (i.e., pages that 521d3576fdSSage Weil * are not yet snapped) and are writing the most recently dirtied 531d3576fdSSage Weil * pages. 541d3576fdSSage Weil * 551d3576fdSSage Weil * Invalidate and so forth must take care to ensure the dirty page 561d3576fdSSage Weil * accounting is preserved. 571d3576fdSSage Weil */ 581d3576fdSSage Weil 592baba250SYehuda Sadeh #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) 602baba250SYehuda Sadeh #define CONGESTION_OFF_THRESH(congestion_kb) \ 612baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) - \ 622baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) >> 2)) 632baba250SYehuda Sadeh 6461600ef8SYan, Zheng static inline struct ceph_snap_context *page_snap_context(struct page *page) 6561600ef8SYan, Zheng { 6661600ef8SYan, Zheng if (PagePrivate(page)) 6761600ef8SYan, Zheng return (void *)page->private; 6861600ef8SYan, Zheng return NULL; 6961600ef8SYan, Zheng } 701d3576fdSSage Weil 711d3576fdSSage Weil /* 721d3576fdSSage Weil * Dirty a page. Optimistically adjust accounting, on the assumption 731d3576fdSSage Weil * that we won't race with invalidate. If we do, readjust. 741d3576fdSSage Weil */ 751d3576fdSSage Weil static int ceph_set_page_dirty(struct page *page) 761d3576fdSSage Weil { 771d3576fdSSage Weil struct address_space *mapping = page->mapping; 781d3576fdSSage Weil struct inode *inode; 791d3576fdSSage Weil struct ceph_inode_info *ci; 801d3576fdSSage Weil struct ceph_snap_context *snapc; 817d6e1f54SSha Zhengju int ret; 821d3576fdSSage Weil 831d3576fdSSage Weil if (unlikely(!mapping)) 841d3576fdSSage Weil return !TestSetPageDirty(page); 851d3576fdSSage Weil 867d6e1f54SSha Zhengju if (PageDirty(page)) { 871d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu -- already dirty\n", 881d3576fdSSage Weil mapping->host, page, page->index); 897d6e1f54SSha Zhengju BUG_ON(!PagePrivate(page)); 901d3576fdSSage Weil return 0; 911d3576fdSSage Weil } 921d3576fdSSage Weil 931d3576fdSSage Weil inode = mapping->host; 941d3576fdSSage Weil ci = ceph_inode(inode); 951d3576fdSSage Weil 961d3576fdSSage Weil /* dirty the head */ 97be655596SSage Weil spin_lock(&ci->i_ceph_lock); 985dda377cSYan, Zheng BUG_ON(ci->i_wr_ref == 0); // caller should hold Fw reference 995dda377cSYan, Zheng if (__ceph_have_pending_cap_snap(ci)) { 1005dda377cSYan, Zheng struct ceph_cap_snap *capsnap = 1015dda377cSYan, Zheng list_last_entry(&ci->i_cap_snaps, 1025dda377cSYan, Zheng struct ceph_cap_snap, 1035dda377cSYan, Zheng ci_item); 1045dda377cSYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 1055dda377cSYan, Zheng capsnap->dirty_pages++; 1065dda377cSYan, Zheng } else { 1075dda377cSYan, Zheng BUG_ON(!ci->i_head_snapc); 1085dda377cSYan, Zheng snapc = ceph_get_snap_context(ci->i_head_snapc); 1091d3576fdSSage Weil ++ci->i_wrbuffer_ref_head; 1105dda377cSYan, Zheng } 1111d3576fdSSage Weil if (ci->i_wrbuffer_ref == 0) 1120444d76aSDave Chinner ihold(inode); 1131d3576fdSSage Weil ++ci->i_wrbuffer_ref; 1141d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu head %d/%d -> %d/%d " 1151d3576fdSSage Weil "snapc %p seq %lld (%d snaps)\n", 1161d3576fdSSage Weil mapping->host, page, page->index, 1171d3576fdSSage Weil ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, 1181d3576fdSSage Weil ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, 1191d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 120be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 1211d3576fdSSage Weil 1221d3576fdSSage Weil /* 1231d3576fdSSage Weil * Reference snap context in page->private. Also set 1241d3576fdSSage Weil * PagePrivate so that we get invalidatepage callback. 1251d3576fdSSage Weil */ 1267d6e1f54SSha Zhengju BUG_ON(PagePrivate(page)); 1271d3576fdSSage Weil page->private = (unsigned long)snapc; 1281d3576fdSSage Weil SetPagePrivate(page); 1291d3576fdSSage Weil 1307d6e1f54SSha Zhengju ret = __set_page_dirty_nobuffers(page); 1317d6e1f54SSha Zhengju WARN_ON(!PageLocked(page)); 1327d6e1f54SSha Zhengju WARN_ON(!page->mapping); 1331d3576fdSSage Weil 1347d6e1f54SSha Zhengju return ret; 1351d3576fdSSage Weil } 1361d3576fdSSage Weil 1371d3576fdSSage Weil /* 1381d3576fdSSage Weil * If we are truncating the full page (i.e. offset == 0), adjust the 1391d3576fdSSage Weil * dirty page counters appropriately. Only called if there is private 1401d3576fdSSage Weil * data on the page. 1411d3576fdSSage Weil */ 142d47992f8SLukas Czerner static void ceph_invalidatepage(struct page *page, unsigned int offset, 143d47992f8SLukas Czerner unsigned int length) 1441d3576fdSSage Weil { 1454ce1e9adSAlexander Beregalov struct inode *inode; 1461d3576fdSSage Weil struct ceph_inode_info *ci; 14761600ef8SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 1481d3576fdSSage Weil 1494ce1e9adSAlexander Beregalov inode = page->mapping->host; 150b150f5c1SMilosz Tanski ci = ceph_inode(inode); 151b150f5c1SMilosz Tanski 15209cbfeafSKirill A. Shutemov if (offset != 0 || length != PAGE_SIZE) { 153b150f5c1SMilosz Tanski dout("%p invalidatepage %p idx %lu partial dirty page %u~%u\n", 154b150f5c1SMilosz Tanski inode, page, page->index, offset, length); 155b150f5c1SMilosz Tanski return; 156b150f5c1SMilosz Tanski } 1574ce1e9adSAlexander Beregalov 15899ccbd22SMilosz Tanski ceph_invalidate_fscache_page(inode, page); 15999ccbd22SMilosz Tanski 160b072d774SYan, Zheng WARN_ON(!PageLocked(page)); 16199ccbd22SMilosz Tanski if (!PagePrivate(page)) 16299ccbd22SMilosz Tanski return; 16399ccbd22SMilosz Tanski 164569d39fcSLukas Czerner dout("%p invalidatepage %p idx %lu full dirty page\n", 165569d39fcSLukas Czerner inode, page, page->index); 166b150f5c1SMilosz Tanski 1671d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 1681d3576fdSSage Weil ceph_put_snap_context(snapc); 1691d3576fdSSage Weil page->private = 0; 1701d3576fdSSage Weil ClearPagePrivate(page); 1711d3576fdSSage Weil } 1721d3576fdSSage Weil 1731d3576fdSSage Weil static int ceph_releasepage(struct page *page, gfp_t g) 1741d3576fdSSage Weil { 175e55f1a18SNeilBrown dout("%p releasepage %p idx %lu (%sdirty)\n", page->mapping->host, 176e55f1a18SNeilBrown page, page->index, PageDirty(page) ? "" : "not "); 17799ccbd22SMilosz Tanski 17899ccbd22SMilosz Tanski /* Can we release the page from the cache? */ 17999ccbd22SMilosz Tanski if (!ceph_release_fscache_page(page, g)) 1801d3576fdSSage Weil return 0; 18199ccbd22SMilosz Tanski 18299ccbd22SMilosz Tanski return !PagePrivate(page); 1831d3576fdSSage Weil } 1841d3576fdSSage Weil 1859b4862ecSJeff Layton /* read a single page, without unlocking it. */ 186dd2bc473SYan, Zheng static int ceph_do_readpage(struct file *filp, struct page *page) 1871d3576fdSSage Weil { 188496ad9aaSAl Viro struct inode *inode = file_inode(filp); 1891d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 190131d7eb4SYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 1919b4862ecSJeff Layton struct ceph_osd_client *osdc = &fsc->client->osdc; 1929b4862ecSJeff Layton struct ceph_osd_request *req; 1939b4862ecSJeff Layton struct ceph_vino vino = ceph_vino(inode); 1941d3576fdSSage Weil int err = 0; 19583701246SYan, Zheng u64 off = page_offset(page); 19609cbfeafSKirill A. Shutemov u64 len = PAGE_SIZE; 1971d3576fdSSage Weil 19883701246SYan, Zheng if (off >= i_size_read(inode)) { 19909cbfeafSKirill A. Shutemov zero_user_segment(page, 0, PAGE_SIZE); 20083701246SYan, Zheng SetPageUptodate(page); 20183701246SYan, Zheng return 0; 20283701246SYan, Zheng } 20399ccbd22SMilosz Tanski 204fcc02d2aSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 20583701246SYan, Zheng /* 206fcc02d2aSYan, Zheng * Uptodate inline data should have been added 207fcc02d2aSYan, Zheng * into page cache while getting Fcr caps. 20883701246SYan, Zheng */ 209fcc02d2aSYan, Zheng if (off == 0) 21083701246SYan, Zheng return -EINVAL; 21109cbfeafSKirill A. Shutemov zero_user_segment(page, 0, PAGE_SIZE); 212fcc02d2aSYan, Zheng SetPageUptodate(page); 213fcc02d2aSYan, Zheng return 0; 214fcc02d2aSYan, Zheng } 21583701246SYan, Zheng 21683701246SYan, Zheng err = ceph_readpage_from_fscache(inode, page); 21799ccbd22SMilosz Tanski if (err == 0) 218dd2bc473SYan, Zheng return -EINPROGRESS; 21999ccbd22SMilosz Tanski 2209b4862ecSJeff Layton dout("readpage ino %llx.%llx file %p off %llu len %llu page %p index %lu\n", 2219b4862ecSJeff Layton vino.ino, vino.snap, filp, off, len, page, page->index); 2229b4862ecSJeff Layton req = ceph_osdc_new_request(osdc, &ci->i_layout, vino, off, &len, 0, 1, 2239b4862ecSJeff Layton CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ, NULL, 2241d3576fdSSage Weil ci->i_truncate_seq, ci->i_truncate_size, 2259b4862ecSJeff Layton false); 2269b4862ecSJeff Layton if (IS_ERR(req)) 2279b4862ecSJeff Layton return PTR_ERR(req); 2289b4862ecSJeff Layton 2299b4862ecSJeff Layton osd_req_op_extent_osd_data_pages(req, 0, &page, len, 0, false, false); 2309b4862ecSJeff Layton 2319b4862ecSJeff Layton err = ceph_osdc_start_request(osdc, req, false); 2329b4862ecSJeff Layton if (!err) 2339b4862ecSJeff Layton err = ceph_osdc_wait_request(osdc, req); 2349b4862ecSJeff Layton 2359b4862ecSJeff Layton ceph_update_read_latency(&fsc->mdsc->metric, req->r_start_latency, 2369b4862ecSJeff Layton req->r_end_latency, err); 2379b4862ecSJeff Layton 2389b4862ecSJeff Layton ceph_osdc_put_request(req); 2399b4862ecSJeff Layton dout("readpage result %d\n", err); 2409b4862ecSJeff Layton 2411d3576fdSSage Weil if (err == -ENOENT) 2421d3576fdSSage Weil err = 0; 2431d3576fdSSage Weil if (err < 0) { 2441d3576fdSSage Weil SetPageError(page); 24518302805SLi Wang ceph_fscache_readpage_cancel(inode, page); 2460b98acd6SIlya Dryomov if (err == -EBLOCKLISTED) 2470b98acd6SIlya Dryomov fsc->blocklisted = true; 2481d3576fdSSage Weil goto out; 24923cd573bSZhang Zhen } 25009cbfeafSKirill A. Shutemov if (err < PAGE_SIZE) 2511d3576fdSSage Weil /* zero fill remainder of page */ 25209cbfeafSKirill A. Shutemov zero_user_segment(page, err, PAGE_SIZE); 25323cd573bSZhang Zhen else 25456f91aadSLi Wang flush_dcache_page(page); 2551d3576fdSSage Weil 25623cd573bSZhang Zhen SetPageUptodate(page); 25799ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 25899ccbd22SMilosz Tanski 2591d3576fdSSage Weil out: 2601d3576fdSSage Weil return err < 0 ? err : 0; 2611d3576fdSSage Weil } 2621d3576fdSSage Weil 2631d3576fdSSage Weil static int ceph_readpage(struct file *filp, struct page *page) 2641d3576fdSSage Weil { 265dd2bc473SYan, Zheng int r = ceph_do_readpage(filp, page); 266dd2bc473SYan, Zheng if (r != -EINPROGRESS) 2671d3576fdSSage Weil unlock_page(page); 268dd2bc473SYan, Zheng else 269dd2bc473SYan, Zheng r = 0; 2701d3576fdSSage Weil return r; 2711d3576fdSSage Weil } 2721d3576fdSSage Weil 2731d3576fdSSage Weil /* 2747c272194SSage Weil * Finish an async read(ahead) op. 2751d3576fdSSage Weil */ 27685e084feSIlya Dryomov static void finish_read(struct ceph_osd_request *req) 2771d3576fdSSage Weil { 2787c272194SSage Weil struct inode *inode = req->r_inode; 27997e27aaaSXiubo Li struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 28087060c10SAlex Elder struct ceph_osd_data *osd_data; 28185e084feSIlya Dryomov int rc = req->r_result <= 0 ? req->r_result : 0; 28285e084feSIlya Dryomov int bytes = req->r_result >= 0 ? req->r_result : 0; 283e0c59487SAlex Elder int num_pages; 2847c272194SSage Weil int i; 2857c272194SSage Weil 2867c272194SSage Weil dout("finish_read %p req %p rc %d bytes %d\n", inode, req, rc, bytes); 2870b98acd6SIlya Dryomov if (rc == -EBLOCKLISTED) 2880b98acd6SIlya Dryomov ceph_inode_to_client(inode)->blocklisted = true; 2897c272194SSage Weil 2907c272194SSage Weil /* unlock all pages, zeroing any data we didn't read */ 291406e2c9fSAlex Elder osd_data = osd_req_op_extent_osd_data(req, 0); 29287060c10SAlex Elder BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 29387060c10SAlex Elder num_pages = calc_pages_for((u64)osd_data->alignment, 29487060c10SAlex Elder (u64)osd_data->length); 295e0c59487SAlex Elder for (i = 0; i < num_pages; i++) { 29687060c10SAlex Elder struct page *page = osd_data->pages[i]; 2977c272194SSage Weil 298368e3585SYan, Zheng if (rc < 0 && rc != -ENOENT) { 299368e3585SYan, Zheng ceph_fscache_readpage_cancel(inode, page); 300f36132a7SLi Wang goto unlock; 301368e3585SYan, Zheng } 30209cbfeafSKirill A. Shutemov if (bytes < (int)PAGE_SIZE) { 3037c272194SSage Weil /* zero (remainder of) page */ 3047c272194SSage Weil int s = bytes < 0 ? 0 : bytes; 30509cbfeafSKirill A. Shutemov zero_user_segment(page, s, PAGE_SIZE); 3067c272194SSage Weil } 3077c272194SSage Weil dout("finish_read %p uptodate %p idx %lu\n", inode, page, 3087c272194SSage Weil page->index); 3097c272194SSage Weil flush_dcache_page(page); 3107c272194SSage Weil SetPageUptodate(page); 31199ccbd22SMilosz Tanski ceph_readpage_to_fscache(inode, page); 312f36132a7SLi Wang unlock: 3137c272194SSage Weil unlock_page(page); 31409cbfeafSKirill A. Shutemov put_page(page); 31509cbfeafSKirill A. Shutemov bytes -= PAGE_SIZE; 3167c272194SSage Weil } 31797e27aaaSXiubo Li 31897e27aaaSXiubo Li ceph_update_read_latency(&fsc->mdsc->metric, req->r_start_latency, 31997e27aaaSXiubo Li req->r_end_latency, rc); 32097e27aaaSXiubo Li 32187060c10SAlex Elder kfree(osd_data->pages); 3227c272194SSage Weil } 3237c272194SSage Weil 3247c272194SSage Weil /* 3257c272194SSage Weil * start an async read(ahead) operation. return nr_pages we submitted 3267c272194SSage Weil * a read for on success, or negative error code. 3277c272194SSage Weil */ 3285d988308SYan, Zheng static int start_read(struct inode *inode, struct ceph_rw_context *rw_ctx, 3295d988308SYan, Zheng struct list_head *page_list, int max) 3307c272194SSage Weil { 3317c272194SSage Weil struct ceph_osd_client *osdc = 3327c272194SSage Weil &ceph_inode_to_client(inode)->client->osdc; 3337c272194SSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 334f86196eaSNikolay Borisov struct page *page = lru_to_page(page_list); 335acead002SAlex Elder struct ceph_vino vino; 3367c272194SSage Weil struct ceph_osd_request *req; 3377c272194SSage Weil u64 off; 3387c272194SSage Weil u64 len; 3397c272194SSage Weil int i; 3401d3576fdSSage Weil struct page **pages; 3417c272194SSage Weil pgoff_t next_index; 3427c272194SSage Weil int nr_pages = 0; 3432b1ac852SYan, Zheng int got = 0; 3442b1ac852SYan, Zheng int ret = 0; 3452b1ac852SYan, Zheng 3465d988308SYan, Zheng if (!rw_ctx) { 3472b1ac852SYan, Zheng /* caller of readpages does not hold buffer and read caps 3482b1ac852SYan, Zheng * (fadvise, madvise and readahead cases) */ 3492b1ac852SYan, Zheng int want = CEPH_CAP_FILE_CACHE; 3505e3ded1bSYan, Zheng ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, 3515e3ded1bSYan, Zheng true, &got); 3522b1ac852SYan, Zheng if (ret < 0) { 3532b1ac852SYan, Zheng dout("start_read %p, error getting cap\n", inode); 3542b1ac852SYan, Zheng } else if (!(got & want)) { 3552b1ac852SYan, Zheng dout("start_read %p, no cache cap\n", inode); 3562b1ac852SYan, Zheng ret = 0; 3572b1ac852SYan, Zheng } 3582b1ac852SYan, Zheng if (ret <= 0) { 3592b1ac852SYan, Zheng if (got) 3602b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 3612b1ac852SYan, Zheng while (!list_empty(page_list)) { 362f86196eaSNikolay Borisov page = lru_to_page(page_list); 3632b1ac852SYan, Zheng list_del(&page->lru); 3642b1ac852SYan, Zheng put_page(page); 3652b1ac852SYan, Zheng } 3662b1ac852SYan, Zheng return ret; 3672b1ac852SYan, Zheng } 3682b1ac852SYan, Zheng } 3697c272194SSage Weil 3706285bc23SAlex Elder off = (u64) page_offset(page); 3717c272194SSage Weil 3727c272194SSage Weil /* count pages */ 3737c272194SSage Weil next_index = page->index; 3747c272194SSage Weil list_for_each_entry_reverse(page, page_list, lru) { 3757c272194SSage Weil if (page->index != next_index) 3767c272194SSage Weil break; 3777c272194SSage Weil nr_pages++; 3787c272194SSage Weil next_index++; 3790d66a487SSage Weil if (max && nr_pages == max) 3800d66a487SSage Weil break; 3817c272194SSage Weil } 38209cbfeafSKirill A. Shutemov len = nr_pages << PAGE_SHIFT; 3837c272194SSage Weil dout("start_read %p nr_pages %d is %lld~%lld\n", inode, nr_pages, 3847c272194SSage Weil off, len); 385acead002SAlex Elder vino = ceph_vino(inode); 386acead002SAlex Elder req = ceph_osdc_new_request(osdc, &ci->i_layout, vino, off, &len, 387715e4cd4SYan, Zheng 0, 1, CEPH_OSD_OP_READ, 388acead002SAlex Elder CEPH_OSD_FLAG_READ, NULL, 3897c272194SSage Weil ci->i_truncate_seq, ci->i_truncate_size, 390acead002SAlex Elder false); 3912b1ac852SYan, Zheng if (IS_ERR(req)) { 3922b1ac852SYan, Zheng ret = PTR_ERR(req); 3932b1ac852SYan, Zheng goto out; 3942b1ac852SYan, Zheng } 3951d3576fdSSage Weil 3961d3576fdSSage Weil /* build page vector */ 397cf7b7e14SAlex Elder nr_pages = calc_pages_for(0, len); 3986da2ec56SKees Cook pages = kmalloc_array(nr_pages, sizeof(*pages), GFP_KERNEL); 3992b1ac852SYan, Zheng if (!pages) { 4007c272194SSage Weil ret = -ENOMEM; 4012b1ac852SYan, Zheng goto out_put; 4022b1ac852SYan, Zheng } 4037c272194SSage Weil for (i = 0; i < nr_pages; ++i) { 4047c272194SSage Weil page = list_entry(page_list->prev, struct page, lru); 4057c272194SSage Weil BUG_ON(PageLocked(page)); 4067c272194SSage Weil list_del(&page->lru); 4071d3576fdSSage Weil 4087c272194SSage Weil dout("start_read %p adding %p idx %lu\n", inode, page, 4097c272194SSage Weil page->index); 4107c272194SSage Weil if (add_to_page_cache_lru(page, &inode->i_data, page->index, 411687265e5SYan, Zheng GFP_KERNEL)) { 412d4d3aa38SMilosz Tanski ceph_fscache_uncache_page(inode, page); 41309cbfeafSKirill A. Shutemov put_page(page); 4147c272194SSage Weil dout("start_read %p add_to_page_cache failed %p\n", 4157c272194SSage Weil inode, page); 4167c272194SSage Weil nr_pages = i; 4171afe4785SYan, Zheng if (nr_pages > 0) { 4181afe4785SYan, Zheng len = nr_pages << PAGE_SHIFT; 419d641df81SYan, Zheng osd_req_op_extent_update(req, 0, len); 4201afe4785SYan, Zheng break; 4211afe4785SYan, Zheng } 4227c272194SSage Weil goto out_pages; 4231d3576fdSSage Weil } 4247c272194SSage Weil pages[i] = page; 4251d3576fdSSage Weil } 426406e2c9fSAlex Elder osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false); 4277c272194SSage Weil req->r_callback = finish_read; 4287c272194SSage Weil req->r_inode = inode; 4297c272194SSage Weil 4307c272194SSage Weil dout("start_read %p starting %p %lld~%lld\n", inode, req, off, len); 4317c272194SSage Weil ret = ceph_osdc_start_request(osdc, req, false); 4327c272194SSage Weil if (ret < 0) 4337c272194SSage Weil goto out_pages; 4347c272194SSage Weil ceph_osdc_put_request(req); 4352b1ac852SYan, Zheng 4362b1ac852SYan, Zheng /* After adding locked pages to page cache, the inode holds cache cap. 4372b1ac852SYan, Zheng * So we can drop our cap refs. */ 4382b1ac852SYan, Zheng if (got) 4392b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 4402b1ac852SYan, Zheng 4417c272194SSage Weil return nr_pages; 4427c272194SSage Weil 4437c272194SSage Weil out_pages: 4441afe4785SYan, Zheng for (i = 0; i < nr_pages; ++i) { 4451afe4785SYan, Zheng ceph_fscache_readpage_cancel(inode, pages[i]); 4461afe4785SYan, Zheng unlock_page(pages[i]); 4471afe4785SYan, Zheng } 4481afe4785SYan, Zheng ceph_put_page_vector(pages, nr_pages, false); 4492b1ac852SYan, Zheng out_put: 4507c272194SSage Weil ceph_osdc_put_request(req); 4512b1ac852SYan, Zheng out: 4522b1ac852SYan, Zheng if (got) 4532b1ac852SYan, Zheng ceph_put_cap_refs(ci, got); 4547c272194SSage Weil return ret; 4551d3576fdSSage Weil } 4561d3576fdSSage Weil 4577c272194SSage Weil 4581d3576fdSSage Weil /* 4591d3576fdSSage Weil * Read multiple pages. Leave pages we don't read + unlock in page_list; 4601d3576fdSSage Weil * the caller (VM) cleans them up. 4611d3576fdSSage Weil */ 4621d3576fdSSage Weil static int ceph_readpages(struct file *file, struct address_space *mapping, 4631d3576fdSSage Weil struct list_head *page_list, unsigned nr_pages) 4641d3576fdSSage Weil { 465496ad9aaSAl Viro struct inode *inode = file_inode(file); 4660d66a487SSage Weil struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 46773737682SChengguang Xu struct ceph_file_info *fi = file->private_data; 4685d988308SYan, Zheng struct ceph_rw_context *rw_ctx; 4691d3576fdSSage Weil int rc = 0; 4700d66a487SSage Weil int max = 0; 4711d3576fdSSage Weil 47283701246SYan, Zheng if (ceph_inode(inode)->i_inline_version != CEPH_INLINE_NONE) 47383701246SYan, Zheng return -EINVAL; 47483701246SYan, Zheng 47599ccbd22SMilosz Tanski rc = ceph_readpages_from_fscache(mapping->host, mapping, page_list, 47699ccbd22SMilosz Tanski &nr_pages); 47799ccbd22SMilosz Tanski 47899ccbd22SMilosz Tanski if (rc == 0) 47999ccbd22SMilosz Tanski goto out; 48099ccbd22SMilosz Tanski 48173737682SChengguang Xu rw_ctx = ceph_find_rw_context(fi); 482aa187926SYan, Zheng max = fsc->mount_options->rsize >> PAGE_SHIFT; 4835d988308SYan, Zheng dout("readpages %p file %p ctx %p nr_pages %d max %d\n", 4845d988308SYan, Zheng inode, file, rw_ctx, nr_pages, max); 4857c272194SSage Weil while (!list_empty(page_list)) { 4865d988308SYan, Zheng rc = start_read(inode, rw_ctx, page_list, max); 4871d3576fdSSage Weil if (rc < 0) 4881d3576fdSSage Weil goto out; 4891d3576fdSSage Weil } 4901d3576fdSSage Weil out: 49176be778bSMilosz Tanski ceph_fscache_readpages_cancel(inode, page_list); 49276be778bSMilosz Tanski 4937c272194SSage Weil dout("readpages %p file %p ret %d\n", inode, file, rc); 4941d3576fdSSage Weil return rc; 4951d3576fdSSage Weil } 4961d3576fdSSage Weil 4971f934b00SYan, Zheng struct ceph_writeback_ctl 4981f934b00SYan, Zheng { 4991f934b00SYan, Zheng loff_t i_size; 5001f934b00SYan, Zheng u64 truncate_size; 5011f934b00SYan, Zheng u32 truncate_seq; 5021f934b00SYan, Zheng bool size_stable; 5032a2d927eSYan, Zheng bool head_snapc; 5041f934b00SYan, Zheng }; 5051f934b00SYan, Zheng 5061d3576fdSSage Weil /* 5071d3576fdSSage Weil * Get ref for the oldest snapc for an inode with dirty data... that is, the 5081d3576fdSSage Weil * only snap context we are allowed to write back. 5091d3576fdSSage Weil */ 5101f934b00SYan, Zheng static struct ceph_snap_context * 51105455e11SYan, Zheng get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, 51205455e11SYan, Zheng struct ceph_snap_context *page_snapc) 5131d3576fdSSage Weil { 5141d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 5151d3576fdSSage Weil struct ceph_snap_context *snapc = NULL; 5161d3576fdSSage Weil struct ceph_cap_snap *capsnap = NULL; 5171d3576fdSSage Weil 518be655596SSage Weil spin_lock(&ci->i_ceph_lock); 5191d3576fdSSage Weil list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 5201d3576fdSSage Weil dout(" cap_snap %p snapc %p has %d dirty pages\n", capsnap, 5211d3576fdSSage Weil capsnap->context, capsnap->dirty_pages); 52205455e11SYan, Zheng if (!capsnap->dirty_pages) 52305455e11SYan, Zheng continue; 52405455e11SYan, Zheng 52505455e11SYan, Zheng /* get i_size, truncate_{seq,size} for page_snapc? */ 52605455e11SYan, Zheng if (snapc && capsnap->context != page_snapc) 52705455e11SYan, Zheng continue; 52805455e11SYan, Zheng 5291f934b00SYan, Zheng if (ctl) { 5301f934b00SYan, Zheng if (capsnap->writing) { 5311f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 5321f934b00SYan, Zheng ctl->size_stable = false; 5331f934b00SYan, Zheng } else { 5341f934b00SYan, Zheng ctl->i_size = capsnap->size; 5351f934b00SYan, Zheng ctl->size_stable = true; 5361f934b00SYan, Zheng } 5371f934b00SYan, Zheng ctl->truncate_size = capsnap->truncate_size; 5381f934b00SYan, Zheng ctl->truncate_seq = capsnap->truncate_seq; 5392a2d927eSYan, Zheng ctl->head_snapc = false; 5401f934b00SYan, Zheng } 54105455e11SYan, Zheng 54205455e11SYan, Zheng if (snapc) 5431d3576fdSSage Weil break; 54405455e11SYan, Zheng 54505455e11SYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 54605455e11SYan, Zheng if (!page_snapc || 54705455e11SYan, Zheng page_snapc == snapc || 54805455e11SYan, Zheng page_snapc->seq > snapc->seq) 54905455e11SYan, Zheng break; 5501d3576fdSSage Weil } 5517d8cb26dSSage Weil if (!snapc && ci->i_wrbuffer_ref_head) { 55280e755feSSage Weil snapc = ceph_get_snap_context(ci->i_head_snapc); 5531d3576fdSSage Weil dout(" head snapc %p has %d dirty pages\n", 5541d3576fdSSage Weil snapc, ci->i_wrbuffer_ref_head); 5551f934b00SYan, Zheng if (ctl) { 5561f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 5571f934b00SYan, Zheng ctl->truncate_size = ci->i_truncate_size; 5581f934b00SYan, Zheng ctl->truncate_seq = ci->i_truncate_seq; 5591f934b00SYan, Zheng ctl->size_stable = false; 5602a2d927eSYan, Zheng ctl->head_snapc = true; 5611f934b00SYan, Zheng } 5621d3576fdSSage Weil } 563be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 5641d3576fdSSage Weil return snapc; 5651d3576fdSSage Weil } 5661d3576fdSSage Weil 5671f934b00SYan, Zheng static u64 get_writepages_data_length(struct inode *inode, 5681f934b00SYan, Zheng struct page *page, u64 start) 5691f934b00SYan, Zheng { 5701f934b00SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 5711f934b00SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 5721f934b00SYan, Zheng struct ceph_cap_snap *capsnap = NULL; 5731f934b00SYan, Zheng u64 end = i_size_read(inode); 5741f934b00SYan, Zheng 5751f934b00SYan, Zheng if (snapc != ci->i_head_snapc) { 5761f934b00SYan, Zheng bool found = false; 5771f934b00SYan, Zheng spin_lock(&ci->i_ceph_lock); 5781f934b00SYan, Zheng list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 5791f934b00SYan, Zheng if (capsnap->context == snapc) { 5801f934b00SYan, Zheng if (!capsnap->writing) 5811f934b00SYan, Zheng end = capsnap->size; 5821f934b00SYan, Zheng found = true; 5831f934b00SYan, Zheng break; 5841f934b00SYan, Zheng } 5851f934b00SYan, Zheng } 5861f934b00SYan, Zheng spin_unlock(&ci->i_ceph_lock); 5871f934b00SYan, Zheng WARN_ON(!found); 5881f934b00SYan, Zheng } 5891f934b00SYan, Zheng if (end > page_offset(page) + PAGE_SIZE) 5901f934b00SYan, Zheng end = page_offset(page) + PAGE_SIZE; 5911f934b00SYan, Zheng return end > start ? end - start : 0; 5921f934b00SYan, Zheng } 5931f934b00SYan, Zheng 5941d3576fdSSage Weil /* 5951d3576fdSSage Weil * Write a single page, but leave the page locked. 5961d3576fdSSage Weil * 597b72b13ebSJeff Layton * If we get a write error, mark the mapping for error, but still adjust the 5981d3576fdSSage Weil * dirty page accounting (i.e., page is no longer dirty). 5991d3576fdSSage Weil */ 6001d3576fdSSage Weil static int writepage_nounlock(struct page *page, struct writeback_control *wbc) 6011d3576fdSSage Weil { 6026390987fSJeff Layton struct inode *inode = page->mapping->host; 6036390987fSJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 6046390987fSJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 6056298a337SSage Weil struct ceph_snap_context *snapc, *oldest; 606fc2744aaSYan, Zheng loff_t page_off = page_offset(page); 6076390987fSJeff Layton int err; 6086390987fSJeff Layton loff_t len = PAGE_SIZE; 6091f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 6106390987fSJeff Layton struct ceph_osd_client *osdc = &fsc->client->osdc; 6116390987fSJeff Layton struct ceph_osd_request *req; 6121d3576fdSSage Weil 6131d3576fdSSage Weil dout("writepage %p idx %lu\n", page, page->index); 6141d3576fdSSage Weil 6151d3576fdSSage Weil /* verify this is a writeable snap context */ 61661600ef8SYan, Zheng snapc = page_snap_context(page); 617d37b1d99SMarkus Elfring if (!snapc) { 6181d3576fdSSage Weil dout("writepage %p page %p not dirty?\n", inode, page); 61943986881SYan, Zheng return 0; 6201d3576fdSSage Weil } 62105455e11SYan, Zheng oldest = get_oldest_context(inode, &ceph_wbc, snapc); 6226298a337SSage Weil if (snapc->seq > oldest->seq) { 6231d3576fdSSage Weil dout("writepage %p page %p snapc %p not writeable - noop\n", 62461600ef8SYan, Zheng inode, page, snapc); 6251d3576fdSSage Weil /* we should only noop if called by kswapd */ 626fa71fefbSYan, Zheng WARN_ON(!(current->flags & PF_MEMALLOC)); 6276298a337SSage Weil ceph_put_snap_context(oldest); 628fa71fefbSYan, Zheng redirty_page_for_writepage(wbc, page); 62943986881SYan, Zheng return 0; 6301d3576fdSSage Weil } 6316298a337SSage Weil ceph_put_snap_context(oldest); 6321d3576fdSSage Weil 6331d3576fdSSage Weil /* is this a partial page at end of file? */ 6341f934b00SYan, Zheng if (page_off >= ceph_wbc.i_size) { 6351f934b00SYan, Zheng dout("%p page eof %llu\n", page, ceph_wbc.i_size); 63605455e11SYan, Zheng page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE); 63743986881SYan, Zheng return 0; 638fc2744aaSYan, Zheng } 63943986881SYan, Zheng 6401f934b00SYan, Zheng if (ceph_wbc.i_size < page_off + len) 6411f934b00SYan, Zheng len = ceph_wbc.i_size - page_off; 6421d3576fdSSage Weil 6436390987fSJeff Layton dout("writepage %p page %p index %lu on %llu~%llu snapc %p seq %lld\n", 6441c0a9c2dSYan, Zheng inode, page, page->index, page_off, len, snapc, snapc->seq); 6451d3576fdSSage Weil 646314c4737SYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 6473d14c5d2SYehuda Sadeh CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) 64809dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 6492baba250SYehuda Sadeh 6501d3576fdSSage Weil set_page_writeback(page); 6516390987fSJeff Layton req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode), page_off, &len, 0, 1, 6526390987fSJeff Layton CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, snapc, 6536390987fSJeff Layton ceph_wbc.truncate_seq, ceph_wbc.truncate_size, 6546390987fSJeff Layton true); 6556390987fSJeff Layton if (IS_ERR(req)) { 6566390987fSJeff Layton redirty_page_for_writepage(wbc, page); 6576390987fSJeff Layton end_page_writeback(page); 6586390987fSJeff Layton return PTR_ERR(req); 6596390987fSJeff Layton } 6606390987fSJeff Layton 6616390987fSJeff Layton /* it may be a short write due to an object boundary */ 6626390987fSJeff Layton WARN_ON_ONCE(len > PAGE_SIZE); 6636390987fSJeff Layton osd_req_op_extent_osd_data_pages(req, 0, &page, len, 0, false, false); 6646390987fSJeff Layton dout("writepage %llu~%llu (%llu bytes)\n", page_off, len, len); 6656390987fSJeff Layton 6666390987fSJeff Layton req->r_mtime = inode->i_mtime; 6676390987fSJeff Layton err = ceph_osdc_start_request(osdc, req, true); 6686390987fSJeff Layton if (!err) 6696390987fSJeff Layton err = ceph_osdc_wait_request(osdc, req); 6706390987fSJeff Layton 6716390987fSJeff Layton ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 6726390987fSJeff Layton req->r_end_latency, err); 6736390987fSJeff Layton 6746390987fSJeff Layton ceph_osdc_put_request(req); 6756390987fSJeff Layton if (err == 0) 6766390987fSJeff Layton err = len; 6776390987fSJeff Layton 6781d3576fdSSage Weil if (err < 0) { 679ad15ec06SYan, Zheng struct writeback_control tmp_wbc; 680ad15ec06SYan, Zheng if (!wbc) 681ad15ec06SYan, Zheng wbc = &tmp_wbc; 682ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 683ad15ec06SYan, Zheng /* killed by SIGKILL */ 684ad15ec06SYan, Zheng dout("writepage interrupted page %p\n", page); 685ad15ec06SYan, Zheng redirty_page_for_writepage(wbc, page); 686ad15ec06SYan, Zheng end_page_writeback(page); 68743986881SYan, Zheng return err; 688ad15ec06SYan, Zheng } 6890b98acd6SIlya Dryomov if (err == -EBLOCKLISTED) 6900b98acd6SIlya Dryomov fsc->blocklisted = true; 691ad15ec06SYan, Zheng dout("writepage setting page/mapping error %d %p\n", 692ad15ec06SYan, Zheng err, page); 6931d3576fdSSage Weil mapping_set_error(&inode->i_data, err); 6941d3576fdSSage Weil wbc->pages_skipped++; 6951d3576fdSSage Weil } else { 6961d3576fdSSage Weil dout("writepage cleaned page %p\n", page); 6971d3576fdSSage Weil err = 0; /* vfs expects us to return 0 */ 6981d3576fdSSage Weil } 6991d3576fdSSage Weil page->private = 0; 7001d3576fdSSage Weil ClearPagePrivate(page); 7011d3576fdSSage Weil end_page_writeback(page); 7021d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 7036298a337SSage Weil ceph_put_snap_context(snapc); /* page's reference */ 704314c4737SYan, Zheng 705314c4737SYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 706314c4737SYan, Zheng CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 707314c4737SYan, Zheng clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 708314c4737SYan, Zheng 7091d3576fdSSage Weil return err; 7101d3576fdSSage Weil } 7111d3576fdSSage Weil 7121d3576fdSSage Weil static int ceph_writepage(struct page *page, struct writeback_control *wbc) 7131d3576fdSSage Weil { 714dbd646a8SYehuda Sadeh int err; 715dbd646a8SYehuda Sadeh struct inode *inode = page->mapping->host; 716dbd646a8SYehuda Sadeh BUG_ON(!inode); 71770b666c3SSage Weil ihold(inode); 718dbd646a8SYehuda Sadeh err = writepage_nounlock(page, wbc); 719ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 720ad15ec06SYan, Zheng /* direct memory reclaimer was killed by SIGKILL. return 0 721ad15ec06SYan, Zheng * to prevent caller from setting mapping/page error */ 722ad15ec06SYan, Zheng err = 0; 723ad15ec06SYan, Zheng } 7241d3576fdSSage Weil unlock_page(page); 725dbd646a8SYehuda Sadeh iput(inode); 7261d3576fdSSage Weil return err; 7271d3576fdSSage Weil } 7281d3576fdSSage Weil 7291d3576fdSSage Weil /* 7301d3576fdSSage Weil * async writeback completion handler. 7311d3576fdSSage Weil * 7321d3576fdSSage Weil * If we get an error, set the mapping error bit, but not the individual 7331d3576fdSSage Weil * page error bits. 7341d3576fdSSage Weil */ 73585e084feSIlya Dryomov static void writepages_finish(struct ceph_osd_request *req) 7361d3576fdSSage Weil { 7371d3576fdSSage Weil struct inode *inode = req->r_inode; 7381d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 73987060c10SAlex Elder struct ceph_osd_data *osd_data; 7401d3576fdSSage Weil struct page *page; 7415b64640cSYan, Zheng int num_pages, total_pages = 0; 7425b64640cSYan, Zheng int i, j; 7435b64640cSYan, Zheng int rc = req->r_result; 7441d3576fdSSage Weil struct ceph_snap_context *snapc = req->r_snapc; 7451d3576fdSSage Weil struct address_space *mapping = inode->i_mapping; 7463d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 7475b64640cSYan, Zheng bool remove_page; 7481d3576fdSSage Weil 7495b64640cSYan, Zheng dout("writepages_finish %p rc %d\n", inode, rc); 75026544c62SJeff Layton if (rc < 0) { 7511d3576fdSSage Weil mapping_set_error(mapping, rc); 75226544c62SJeff Layton ceph_set_error_write(ci); 7530b98acd6SIlya Dryomov if (rc == -EBLOCKLISTED) 7540b98acd6SIlya Dryomov fsc->blocklisted = true; 75526544c62SJeff Layton } else { 75626544c62SJeff Layton ceph_clear_error_write(ci); 75726544c62SJeff Layton } 758e63dc5c7SYehuda Sadeh 75997e27aaaSXiubo Li ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 76097e27aaaSXiubo Li req->r_end_latency, rc); 76197e27aaaSXiubo Li 762e63dc5c7SYehuda Sadeh /* 763e63dc5c7SYehuda Sadeh * We lost the cache cap, need to truncate the page before 764e63dc5c7SYehuda Sadeh * it is unlocked, otherwise we'd truncate it later in the 765e63dc5c7SYehuda Sadeh * page truncation thread, possibly losing some data that 766e63dc5c7SYehuda Sadeh * raced its way in 767e63dc5c7SYehuda Sadeh */ 7685b64640cSYan, Zheng remove_page = !(ceph_caps_issued(ci) & 7695b64640cSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); 7705b64640cSYan, Zheng 7715b64640cSYan, Zheng /* clean all pages */ 7725b64640cSYan, Zheng for (i = 0; i < req->r_num_ops; i++) { 7735b64640cSYan, Zheng if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) 7745b64640cSYan, Zheng break; 7755b64640cSYan, Zheng 7765b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, i); 7775b64640cSYan, Zheng BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 7785b64640cSYan, Zheng num_pages = calc_pages_for((u64)osd_data->alignment, 7795b64640cSYan, Zheng (u64)osd_data->length); 7805b64640cSYan, Zheng total_pages += num_pages; 7815b64640cSYan, Zheng for (j = 0; j < num_pages; j++) { 7825b64640cSYan, Zheng page = osd_data->pages[j]; 7835b64640cSYan, Zheng BUG_ON(!page); 7845b64640cSYan, Zheng WARN_ON(!PageUptodate(page)); 7855b64640cSYan, Zheng 7865b64640cSYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 7875b64640cSYan, Zheng CONGESTION_OFF_THRESH( 7885b64640cSYan, Zheng fsc->mount_options->congestion_kb)) 78909dc9fc2SJan Kara clear_bdi_congested(inode_to_bdi(inode), 7905b64640cSYan, Zheng BLK_RW_ASYNC); 7915b64640cSYan, Zheng 7925b64640cSYan, Zheng ceph_put_snap_context(page_snap_context(page)); 7935b64640cSYan, Zheng page->private = 0; 7945b64640cSYan, Zheng ClearPagePrivate(page); 7955b64640cSYan, Zheng dout("unlocking %p\n", page); 7965b64640cSYan, Zheng end_page_writeback(page); 7975b64640cSYan, Zheng 7985b64640cSYan, Zheng if (remove_page) 7995b64640cSYan, Zheng generic_error_remove_page(inode->i_mapping, 8005b64640cSYan, Zheng page); 801e63dc5c7SYehuda Sadeh 8021d3576fdSSage Weil unlock_page(page); 8031d3576fdSSage Weil } 8045b64640cSYan, Zheng dout("writepages_finish %p wrote %llu bytes cleaned %d pages\n", 8055b64640cSYan, Zheng inode, osd_data->length, rc >= 0 ? num_pages : 0); 8061d3576fdSSage Weil 80796ac9158SJohn Hubbard release_pages(osd_data->pages, num_pages); 8085b64640cSYan, Zheng } 8095b64640cSYan, Zheng 8105b64640cSYan, Zheng ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); 8115b64640cSYan, Zheng 8125b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, 0); 81387060c10SAlex Elder if (osd_data->pages_from_pool) 814a0102bdaSJeff Layton mempool_free(osd_data->pages, ceph_wb_pagevec_pool); 8151d3576fdSSage Weil else 81687060c10SAlex Elder kfree(osd_data->pages); 8171d3576fdSSage Weil ceph_osdc_put_request(req); 8181d3576fdSSage Weil } 8191d3576fdSSage Weil 8201d3576fdSSage Weil /* 8211d3576fdSSage Weil * initiate async writeback 8221d3576fdSSage Weil */ 8231d3576fdSSage Weil static int ceph_writepages_start(struct address_space *mapping, 8241d3576fdSSage Weil struct writeback_control *wbc) 8251d3576fdSSage Weil { 8261d3576fdSSage Weil struct inode *inode = mapping->host; 8271d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 828fc2744aaSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 829fc2744aaSYan, Zheng struct ceph_vino vino = ceph_vino(inode); 8302a2d927eSYan, Zheng pgoff_t index, start_index, end = -1; 83180e755feSSage Weil struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; 8321d3576fdSSage Weil struct pagevec pvec; 8331d3576fdSSage Weil int rc = 0; 83493407472SFabian Frederick unsigned int wsize = i_blocksize(inode); 8351d3576fdSSage Weil struct ceph_osd_request *req = NULL; 8361f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 837590e9d98SYan, Zheng bool should_loop, range_whole = false; 838af9cc401SYan, Zheng bool done = false; 8391d3576fdSSage Weil 8403fb99d48SYanhu Cao dout("writepages_start %p (mode=%s)\n", inode, 8411d3576fdSSage Weil wbc->sync_mode == WB_SYNC_NONE ? "NONE" : 8421d3576fdSSage Weil (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); 8431d3576fdSSage Weil 84452953d55SSeraphime Kirkovski if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { 8456c93df5dSYan, Zheng if (ci->i_wrbuffer_ref > 0) { 8466c93df5dSYan, Zheng pr_warn_ratelimited( 8476c93df5dSYan, Zheng "writepage_start %p %lld forced umount\n", 8486c93df5dSYan, Zheng inode, ceph_ino(inode)); 8496c93df5dSYan, Zheng } 850a341d4dfSYan, Zheng mapping_set_error(mapping, -EIO); 8511d3576fdSSage Weil return -EIO; /* we're in a forced umount, don't write! */ 8521d3576fdSSage Weil } 85395cca2b4SYan, Zheng if (fsc->mount_options->wsize < wsize) 8543d14c5d2SYehuda Sadeh wsize = fsc->mount_options->wsize; 8551d3576fdSSage Weil 85686679820SMel Gorman pagevec_init(&pvec); 8571d3576fdSSage Weil 858590e9d98SYan, Zheng start_index = wbc->range_cyclic ? mapping->writeback_index : 0; 859590e9d98SYan, Zheng index = start_index; 8601d3576fdSSage Weil 8611d3576fdSSage Weil retry: 8621d3576fdSSage Weil /* find oldest snap context with dirty data */ 86305455e11SYan, Zheng snapc = get_oldest_context(inode, &ceph_wbc, NULL); 8641d3576fdSSage Weil if (!snapc) { 8651d3576fdSSage Weil /* hmm, why does writepages get called when there 8661d3576fdSSage Weil is no dirty data? */ 8671d3576fdSSage Weil dout(" no snap context with dirty data?\n"); 8681d3576fdSSage Weil goto out; 8691d3576fdSSage Weil } 8701d3576fdSSage Weil dout(" oldest snapc is %p seq %lld (%d snaps)\n", 8711d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 872fc2744aaSYan, Zheng 8732a2d927eSYan, Zheng should_loop = false; 8742a2d927eSYan, Zheng if (ceph_wbc.head_snapc && snapc != last_snapc) { 8752a2d927eSYan, Zheng /* where to start/end? */ 8762a2d927eSYan, Zheng if (wbc->range_cyclic) { 8772a2d927eSYan, Zheng index = start_index; 8782a2d927eSYan, Zheng end = -1; 8792a2d927eSYan, Zheng if (index > 0) 8802a2d927eSYan, Zheng should_loop = true; 8812a2d927eSYan, Zheng dout(" cyclic, start at %lu\n", index); 8822a2d927eSYan, Zheng } else { 8832a2d927eSYan, Zheng index = wbc->range_start >> PAGE_SHIFT; 8842a2d927eSYan, Zheng end = wbc->range_end >> PAGE_SHIFT; 8852a2d927eSYan, Zheng if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 8862a2d927eSYan, Zheng range_whole = true; 8872a2d927eSYan, Zheng dout(" not cyclic, %lu to %lu\n", index, end); 8881d3576fdSSage Weil } 8892a2d927eSYan, Zheng } else if (!ceph_wbc.head_snapc) { 8902a2d927eSYan, Zheng /* Do not respect wbc->range_{start,end}. Dirty pages 8912a2d927eSYan, Zheng * in that range can be associated with newer snapc. 8922a2d927eSYan, Zheng * They are not writeable until we write all dirty pages 8932a2d927eSYan, Zheng * associated with 'snapc' get written */ 8941582af2eSYan, Zheng if (index > 0) 8952a2d927eSYan, Zheng should_loop = true; 8962a2d927eSYan, Zheng dout(" non-head snapc, range whole\n"); 8972a2d927eSYan, Zheng } 8982a2d927eSYan, Zheng 8992a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 9001d3576fdSSage Weil last_snapc = snapc; 9011d3576fdSSage Weil 902af9cc401SYan, Zheng while (!done && index <= end) { 9035b64640cSYan, Zheng int num_ops = 0, op_idx; 9040e5ecac7SYan, Zheng unsigned i, pvec_pages, max_pages, locked_pages = 0; 9055b64640cSYan, Zheng struct page **pages = NULL, **data_pages; 9061d3576fdSSage Weil struct page *page; 9070e5ecac7SYan, Zheng pgoff_t strip_unit_end = 0; 9085b64640cSYan, Zheng u64 offset = 0, len = 0; 909a0102bdaSJeff Layton bool from_pool = false; 9101d3576fdSSage Weil 9110e5ecac7SYan, Zheng max_pages = wsize >> PAGE_SHIFT; 9121d3576fdSSage Weil 9131d3576fdSSage Weil get_more_pages: 9142e169296SJeff Layton pvec_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, 9152e169296SJeff Layton end, PAGECACHE_TAG_DIRTY); 9160ed75fc8SJan Kara dout("pagevec_lookup_range_tag got %d\n", pvec_pages); 9171d3576fdSSage Weil if (!pvec_pages && !locked_pages) 9181d3576fdSSage Weil break; 9191d3576fdSSage Weil for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { 9201d3576fdSSage Weil page = pvec.pages[i]; 9211d3576fdSSage Weil dout("? %p idx %lu\n", page, page->index); 9221d3576fdSSage Weil if (locked_pages == 0) 9231d3576fdSSage Weil lock_page(page); /* first page */ 9241d3576fdSSage Weil else if (!trylock_page(page)) 9251d3576fdSSage Weil break; 9261d3576fdSSage Weil 9271d3576fdSSage Weil /* only dirty pages, or our accounting breaks */ 9281d3576fdSSage Weil if (unlikely(!PageDirty(page)) || 9291d3576fdSSage Weil unlikely(page->mapping != mapping)) { 9301d3576fdSSage Weil dout("!dirty or !mapping %p\n", page); 9311d3576fdSSage Weil unlock_page(page); 9320713e5f2SYan, Zheng continue; 9331d3576fdSSage Weil } 934af9cc401SYan, Zheng /* only if matching snap context */ 935af9cc401SYan, Zheng pgsnapc = page_snap_context(page); 936af9cc401SYan, Zheng if (pgsnapc != snapc) { 937af9cc401SYan, Zheng dout("page snapc %p %lld != oldest %p %lld\n", 938af9cc401SYan, Zheng pgsnapc, pgsnapc->seq, snapc, snapc->seq); 9391582af2eSYan, Zheng if (!should_loop && 9401582af2eSYan, Zheng !ceph_wbc.head_snapc && 9411582af2eSYan, Zheng wbc->sync_mode != WB_SYNC_NONE) 9421582af2eSYan, Zheng should_loop = true; 9431d3576fdSSage Weil unlock_page(page); 944af9cc401SYan, Zheng continue; 9451d3576fdSSage Weil } 9461f934b00SYan, Zheng if (page_offset(page) >= ceph_wbc.i_size) { 9471f934b00SYan, Zheng dout("%p page eof %llu\n", 9481f934b00SYan, Zheng page, ceph_wbc.i_size); 949c95f1c5fSErqi Chen if ((ceph_wbc.size_stable || 950c95f1c5fSErqi Chen page_offset(page) >= i_size_read(inode)) && 951c95f1c5fSErqi Chen clear_page_dirty_for_io(page)) 952af9cc401SYan, Zheng mapping->a_ops->invalidatepage(page, 953af9cc401SYan, Zheng 0, PAGE_SIZE); 954af9cc401SYan, Zheng unlock_page(page); 955af9cc401SYan, Zheng continue; 956af9cc401SYan, Zheng } 957af9cc401SYan, Zheng if (strip_unit_end && (page->index > strip_unit_end)) { 958af9cc401SYan, Zheng dout("end of strip unit %p\n", page); 9591d3576fdSSage Weil unlock_page(page); 9601d3576fdSSage Weil break; 9611d3576fdSSage Weil } 9621d3576fdSSage Weil if (PageWriteback(page)) { 9630713e5f2SYan, Zheng if (wbc->sync_mode == WB_SYNC_NONE) { 9641d3576fdSSage Weil dout("%p under writeback\n", page); 9651d3576fdSSage Weil unlock_page(page); 9660713e5f2SYan, Zheng continue; 9670713e5f2SYan, Zheng } 9680713e5f2SYan, Zheng dout("waiting on writeback %p\n", page); 9690713e5f2SYan, Zheng wait_on_page_writeback(page); 9701d3576fdSSage Weil } 9711d3576fdSSage Weil 9721d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) { 9731d3576fdSSage Weil dout("%p !clear_page_dirty_for_io\n", page); 9741d3576fdSSage Weil unlock_page(page); 9750713e5f2SYan, Zheng continue; 9761d3576fdSSage Weil } 9771d3576fdSSage Weil 978e5975c7cSAlex Elder /* 979e5975c7cSAlex Elder * We have something to write. If this is 980e5975c7cSAlex Elder * the first locked page this time through, 9815b64640cSYan, Zheng * calculate max possinle write size and 9825b64640cSYan, Zheng * allocate a page array 983e5975c7cSAlex Elder */ 9841d3576fdSSage Weil if (locked_pages == 0) { 9855b64640cSYan, Zheng u64 objnum; 9865b64640cSYan, Zheng u64 objoff; 987dccbf080SIlya Dryomov u32 xlen; 9885b64640cSYan, Zheng 9891d3576fdSSage Weil /* prepare async write request */ 9906285bc23SAlex Elder offset = (u64)page_offset(page); 991dccbf080SIlya Dryomov ceph_calc_file_object_mapping(&ci->i_layout, 992dccbf080SIlya Dryomov offset, wsize, 9935b64640cSYan, Zheng &objnum, &objoff, 994dccbf080SIlya Dryomov &xlen); 995dccbf080SIlya Dryomov len = xlen; 9968c71897bSHenry C Chang 9973fb99d48SYanhu Cao num_ops = 1; 9985b64640cSYan, Zheng strip_unit_end = page->index + 99909cbfeafSKirill A. Shutemov ((len - 1) >> PAGE_SHIFT); 1000715e4cd4SYan, Zheng 10015b64640cSYan, Zheng BUG_ON(pages); 100288486957SAlex Elder max_pages = calc_pages_for(0, (u64)len); 10036da2ec56SKees Cook pages = kmalloc_array(max_pages, 10046da2ec56SKees Cook sizeof(*pages), 1005fc2744aaSYan, Zheng GFP_NOFS); 100688486957SAlex Elder if (!pages) { 1007a0102bdaSJeff Layton from_pool = true; 1008a0102bdaSJeff Layton pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); 1009e5975c7cSAlex Elder BUG_ON(!pages); 101088486957SAlex Elder } 10115b64640cSYan, Zheng 10125b64640cSYan, Zheng len = 0; 10135b64640cSYan, Zheng } else if (page->index != 101409cbfeafSKirill A. Shutemov (offset + len) >> PAGE_SHIFT) { 1015a0102bdaSJeff Layton if (num_ops >= (from_pool ? CEPH_OSD_SLAB_OPS : 10165b64640cSYan, Zheng CEPH_OSD_MAX_OPS)) { 10175b64640cSYan, Zheng redirty_page_for_writepage(wbc, page); 10185b64640cSYan, Zheng unlock_page(page); 10195b64640cSYan, Zheng break; 10205b64640cSYan, Zheng } 10215b64640cSYan, Zheng 10225b64640cSYan, Zheng num_ops++; 10235b64640cSYan, Zheng offset = (u64)page_offset(page); 10245b64640cSYan, Zheng len = 0; 10251d3576fdSSage Weil } 10261d3576fdSSage Weil 10271d3576fdSSage Weil /* note position of first page in pvec */ 10281d3576fdSSage Weil dout("%p will write page %p idx %lu\n", 10291d3576fdSSage Weil inode, page, page->index); 10302baba250SYehuda Sadeh 10315b64640cSYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 10325b64640cSYan, Zheng CONGESTION_ON_THRESH( 10333d14c5d2SYehuda Sadeh fsc->mount_options->congestion_kb)) { 103409dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), 1035213c99eeSSage Weil BLK_RW_ASYNC); 10362baba250SYehuda Sadeh } 10372baba250SYehuda Sadeh 10380713e5f2SYan, Zheng 10390713e5f2SYan, Zheng pages[locked_pages++] = page; 10400713e5f2SYan, Zheng pvec.pages[i] = NULL; 10410713e5f2SYan, Zheng 104209cbfeafSKirill A. Shutemov len += PAGE_SIZE; 10431d3576fdSSage Weil } 10441d3576fdSSage Weil 10451d3576fdSSage Weil /* did we get anything? */ 10461d3576fdSSage Weil if (!locked_pages) 10471d3576fdSSage Weil goto release_pvec_pages; 10481d3576fdSSage Weil if (i) { 10490713e5f2SYan, Zheng unsigned j, n = 0; 10500713e5f2SYan, Zheng /* shift unused page to beginning of pvec */ 10510713e5f2SYan, Zheng for (j = 0; j < pvec_pages; j++) { 10520713e5f2SYan, Zheng if (!pvec.pages[j]) 10530713e5f2SYan, Zheng continue; 10540713e5f2SYan, Zheng if (n < j) 10550713e5f2SYan, Zheng pvec.pages[n] = pvec.pages[j]; 10560713e5f2SYan, Zheng n++; 10570713e5f2SYan, Zheng } 10580713e5f2SYan, Zheng pvec.nr = n; 10591d3576fdSSage Weil 10601d3576fdSSage Weil if (pvec_pages && i == pvec_pages && 10611d3576fdSSage Weil locked_pages < max_pages) { 10621d3576fdSSage Weil dout("reached end pvec, trying for more\n"); 10630713e5f2SYan, Zheng pagevec_release(&pvec); 10641d3576fdSSage Weil goto get_more_pages; 10651d3576fdSSage Weil } 10661d3576fdSSage Weil } 10671d3576fdSSage Weil 10685b64640cSYan, Zheng new_request: 1069e5975c7cSAlex Elder offset = page_offset(pages[0]); 10705b64640cSYan, Zheng len = wsize; 10715b64640cSYan, Zheng 10725b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 10735b64640cSYan, Zheng &ci->i_layout, vino, 10745b64640cSYan, Zheng offset, &len, 0, num_ops, 10751f934b00SYan, Zheng CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 10761f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 10771f934b00SYan, Zheng ceph_wbc.truncate_size, false); 10785b64640cSYan, Zheng if (IS_ERR(req)) { 10795b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 10805b64640cSYan, Zheng &ci->i_layout, vino, 10815b64640cSYan, Zheng offset, &len, 0, 10825b64640cSYan, Zheng min(num_ops, 10835b64640cSYan, Zheng CEPH_OSD_SLAB_OPS), 10845b64640cSYan, Zheng CEPH_OSD_OP_WRITE, 108554ea0046SIlya Dryomov CEPH_OSD_FLAG_WRITE, 10861f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 10871f934b00SYan, Zheng ceph_wbc.truncate_size, true); 10885b64640cSYan, Zheng BUG_ON(IS_ERR(req)); 10895b64640cSYan, Zheng } 10905b64640cSYan, Zheng BUG_ON(len < page_offset(pages[locked_pages - 1]) + 109109cbfeafSKirill A. Shutemov PAGE_SIZE - offset); 10925b64640cSYan, Zheng 10935b64640cSYan, Zheng req->r_callback = writepages_finish; 10945b64640cSYan, Zheng req->r_inode = inode; 10955b64640cSYan, Zheng 10965b64640cSYan, Zheng /* Format the osd request message and submit the write */ 10975b64640cSYan, Zheng len = 0; 10985b64640cSYan, Zheng data_pages = pages; 10995b64640cSYan, Zheng op_idx = 0; 11005b64640cSYan, Zheng for (i = 0; i < locked_pages; i++) { 11015b64640cSYan, Zheng u64 cur_offset = page_offset(pages[i]); 11025b64640cSYan, Zheng if (offset + len != cur_offset) { 11033fb99d48SYanhu Cao if (op_idx + 1 == req->r_num_ops) 11045b64640cSYan, Zheng break; 11055b64640cSYan, Zheng osd_req_op_extent_dup_last(req, op_idx, 11065b64640cSYan, Zheng cur_offset - offset); 11075b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", 11085b64640cSYan, Zheng offset, len); 11095b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, 11105b64640cSYan, Zheng data_pages, len, 0, 1111a0102bdaSJeff Layton from_pool, false); 11125b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 11135b64640cSYan, Zheng 11145b64640cSYan, Zheng len = 0; 11155b64640cSYan, Zheng offset = cur_offset; 11165b64640cSYan, Zheng data_pages = pages + i; 11175b64640cSYan, Zheng op_idx++; 11185b64640cSYan, Zheng } 11195b64640cSYan, Zheng 11205b64640cSYan, Zheng set_page_writeback(pages[i]); 112109cbfeafSKirill A. Shutemov len += PAGE_SIZE; 11225b64640cSYan, Zheng } 11235b64640cSYan, Zheng 11241f934b00SYan, Zheng if (ceph_wbc.size_stable) { 11251f934b00SYan, Zheng len = min(len, ceph_wbc.i_size - offset); 11265b64640cSYan, Zheng } else if (i == locked_pages) { 1127e1966b49SYan, Zheng /* writepages_finish() clears writeback pages 1128e1966b49SYan, Zheng * according to the data length, so make sure 1129e1966b49SYan, Zheng * data length covers all locked pages */ 113009cbfeafSKirill A. Shutemov u64 min_len = len + 1 - PAGE_SIZE; 11311f934b00SYan, Zheng len = get_writepages_data_length(inode, pages[i - 1], 11321f934b00SYan, Zheng offset); 11335b64640cSYan, Zheng len = max(len, min_len); 1134e1966b49SYan, Zheng } 11355b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", offset, len); 11361d3576fdSSage Weil 11375b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, data_pages, len, 1138a0102bdaSJeff Layton 0, from_pool, false); 11395b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 1140e5975c7cSAlex Elder 11415b64640cSYan, Zheng BUG_ON(op_idx + 1 != req->r_num_ops); 11425b64640cSYan, Zheng 1143a0102bdaSJeff Layton from_pool = false; 11445b64640cSYan, Zheng if (i < locked_pages) { 11455b64640cSYan, Zheng BUG_ON(num_ops <= req->r_num_ops); 11465b64640cSYan, Zheng num_ops -= req->r_num_ops; 11475b64640cSYan, Zheng locked_pages -= i; 1148e5975c7cSAlex Elder 11495b64640cSYan, Zheng /* allocate new pages array for next request */ 11505b64640cSYan, Zheng data_pages = pages; 11516da2ec56SKees Cook pages = kmalloc_array(locked_pages, sizeof(*pages), 11525b64640cSYan, Zheng GFP_NOFS); 11535b64640cSYan, Zheng if (!pages) { 1154a0102bdaSJeff Layton from_pool = true; 1155a0102bdaSJeff Layton pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); 11565b64640cSYan, Zheng BUG_ON(!pages); 11575b64640cSYan, Zheng } 11585b64640cSYan, Zheng memcpy(pages, data_pages + i, 11595b64640cSYan, Zheng locked_pages * sizeof(*pages)); 11605b64640cSYan, Zheng memset(data_pages + i, 0, 11615b64640cSYan, Zheng locked_pages * sizeof(*pages)); 11625b64640cSYan, Zheng } else { 11635b64640cSYan, Zheng BUG_ON(num_ops != req->r_num_ops); 11645b64640cSYan, Zheng index = pages[i - 1]->index + 1; 11655b64640cSYan, Zheng /* request message now owns the pages array */ 11665b64640cSYan, Zheng pages = NULL; 11675b64640cSYan, Zheng } 1168e5975c7cSAlex Elder 1169fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 11709d6fcb08SSage Weil rc = ceph_osdc_start_request(&fsc->client->osdc, req, true); 11719d6fcb08SSage Weil BUG_ON(rc); 11721d3576fdSSage Weil req = NULL; 11731d3576fdSSage Weil 11745b64640cSYan, Zheng wbc->nr_to_write -= i; 11755b64640cSYan, Zheng if (pages) 11765b64640cSYan, Zheng goto new_request; 11775b64640cSYan, Zheng 11782a2d927eSYan, Zheng /* 11792a2d927eSYan, Zheng * We stop writing back only if we are not doing 11802a2d927eSYan, Zheng * integrity sync. In case of integrity sync we have to 11812a2d927eSYan, Zheng * keep going until we have written all the pages 11822a2d927eSYan, Zheng * we tagged for writeback prior to entering this loop. 11832a2d927eSYan, Zheng */ 11842a2d927eSYan, Zheng if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) 1185af9cc401SYan, Zheng done = true; 11861d3576fdSSage Weil 11871d3576fdSSage Weil release_pvec_pages: 11881d3576fdSSage Weil dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, 11891d3576fdSSage Weil pvec.nr ? pvec.pages[0] : NULL); 11901d3576fdSSage Weil pagevec_release(&pvec); 11911d3576fdSSage Weil } 11921d3576fdSSage Weil 11931d3576fdSSage Weil if (should_loop && !done) { 11941d3576fdSSage Weil /* more to do; loop back to beginning of file */ 11951d3576fdSSage Weil dout("writepages looping back to beginning of file\n"); 11962a2d927eSYan, Zheng end = start_index - 1; /* OK even when start_index == 0 */ 1197f275635eSYan, Zheng 1198f275635eSYan, Zheng /* to write dirty pages associated with next snapc, 1199f275635eSYan, Zheng * we need to wait until current writes complete */ 1200f275635eSYan, Zheng if (wbc->sync_mode != WB_SYNC_NONE && 1201f275635eSYan, Zheng start_index == 0 && /* all dirty pages were checked */ 1202f275635eSYan, Zheng !ceph_wbc.head_snapc) { 1203f275635eSYan, Zheng struct page *page; 1204f275635eSYan, Zheng unsigned i, nr; 1205f275635eSYan, Zheng index = 0; 1206f275635eSYan, Zheng while ((index <= end) && 1207f275635eSYan, Zheng (nr = pagevec_lookup_tag(&pvec, mapping, &index, 120867fd707fSJan Kara PAGECACHE_TAG_WRITEBACK))) { 1209f275635eSYan, Zheng for (i = 0; i < nr; i++) { 1210f275635eSYan, Zheng page = pvec.pages[i]; 1211f275635eSYan, Zheng if (page_snap_context(page) != snapc) 1212f275635eSYan, Zheng continue; 1213f275635eSYan, Zheng wait_on_page_writeback(page); 1214f275635eSYan, Zheng } 1215f275635eSYan, Zheng pagevec_release(&pvec); 1216f275635eSYan, Zheng cond_resched(); 1217f275635eSYan, Zheng } 1218f275635eSYan, Zheng } 1219f275635eSYan, Zheng 12202a2d927eSYan, Zheng start_index = 0; 12211d3576fdSSage Weil index = 0; 12221d3576fdSSage Weil goto retry; 12231d3576fdSSage Weil } 12241d3576fdSSage Weil 12251d3576fdSSage Weil if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 12261d3576fdSSage Weil mapping->writeback_index = index; 12271d3576fdSSage Weil 12281d3576fdSSage Weil out: 12291d3576fdSSage Weil ceph_osdc_put_request(req); 12302a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 12312a2d927eSYan, Zheng dout("writepages dend - startone, rc = %d\n", rc); 12321d3576fdSSage Weil return rc; 12331d3576fdSSage Weil } 12341d3576fdSSage Weil 12351d3576fdSSage Weil 12361d3576fdSSage Weil 12371d3576fdSSage Weil /* 12381d3576fdSSage Weil * See if a given @snapc is either writeable, or already written. 12391d3576fdSSage Weil */ 12401d3576fdSSage Weil static int context_is_writeable_or_written(struct inode *inode, 12411d3576fdSSage Weil struct ceph_snap_context *snapc) 12421d3576fdSSage Weil { 124305455e11SYan, Zheng struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL); 12446298a337SSage Weil int ret = !oldest || snapc->seq <= oldest->seq; 12456298a337SSage Weil 12466298a337SSage Weil ceph_put_snap_context(oldest); 12476298a337SSage Weil return ret; 12481d3576fdSSage Weil } 12491d3576fdSSage Weil 125018d620f0SJeff Layton /** 125118d620f0SJeff Layton * ceph_find_incompatible - find an incompatible context and return it 125218d620f0SJeff Layton * @page: page being dirtied 125318d620f0SJeff Layton * 125418d620f0SJeff Layton * We are only allowed to write into/dirty a page if the page is 125518d620f0SJeff Layton * clean, or already dirty within the same snap context. Returns a 125618d620f0SJeff Layton * conflicting context if there is one, NULL if there isn't, or a 125718d620f0SJeff Layton * negative error code on other errors. 125818d620f0SJeff Layton * 125918d620f0SJeff Layton * Must be called with page lock held. 126018d620f0SJeff Layton */ 126118d620f0SJeff Layton static struct ceph_snap_context * 1262d45156bfSJeff Layton ceph_find_incompatible(struct page *page) 126318d620f0SJeff Layton { 1264d45156bfSJeff Layton struct inode *inode = page->mapping->host; 126518d620f0SJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 126618d620f0SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 126718d620f0SJeff Layton 126818d620f0SJeff Layton if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { 126918d620f0SJeff Layton dout(" page %p forced umount\n", page); 127018d620f0SJeff Layton return ERR_PTR(-EIO); 127118d620f0SJeff Layton } 127218d620f0SJeff Layton 127318d620f0SJeff Layton for (;;) { 127418d620f0SJeff Layton struct ceph_snap_context *snapc, *oldest; 127518d620f0SJeff Layton 127618d620f0SJeff Layton wait_on_page_writeback(page); 127718d620f0SJeff Layton 127818d620f0SJeff Layton snapc = page_snap_context(page); 127918d620f0SJeff Layton if (!snapc || snapc == ci->i_head_snapc) 128018d620f0SJeff Layton break; 128118d620f0SJeff Layton 128218d620f0SJeff Layton /* 128318d620f0SJeff Layton * this page is already dirty in another (older) snap 128418d620f0SJeff Layton * context! is it writeable now? 128518d620f0SJeff Layton */ 128618d620f0SJeff Layton oldest = get_oldest_context(inode, NULL, NULL); 128718d620f0SJeff Layton if (snapc->seq > oldest->seq) { 128818d620f0SJeff Layton /* not writeable -- return it for the caller to deal with */ 128918d620f0SJeff Layton ceph_put_snap_context(oldest); 129018d620f0SJeff Layton dout(" page %p snapc %p not current or oldest\n", page, snapc); 129118d620f0SJeff Layton return ceph_get_snap_context(snapc); 129218d620f0SJeff Layton } 129318d620f0SJeff Layton ceph_put_snap_context(oldest); 129418d620f0SJeff Layton 129518d620f0SJeff Layton /* yay, writeable, do it now (without dropping page lock) */ 129618d620f0SJeff Layton dout(" page %p snapc %p not current, but oldest\n", page, snapc); 129718d620f0SJeff Layton if (clear_page_dirty_for_io(page)) { 129818d620f0SJeff Layton int r = writepage_nounlock(page, NULL); 129918d620f0SJeff Layton if (r < 0) 130018d620f0SJeff Layton return ERR_PTR(r); 130118d620f0SJeff Layton } 130218d620f0SJeff Layton } 130318d620f0SJeff Layton return NULL; 130418d620f0SJeff Layton } 130518d620f0SJeff Layton 13061d3576fdSSage Weil /* 13071d3576fdSSage Weil * We are only allowed to write into/dirty the page if the page is 13081d3576fdSSage Weil * clean, or already dirty within the same snap context. 13098f883c24SSage Weil * 13108f883c24SSage Weil * called with page locked. 13118f883c24SSage Weil * return success with page locked, 13128f883c24SSage Weil * or any failure (incl -EAGAIN) with page unlocked. 13131d3576fdSSage Weil */ 13144af6b225SYehuda Sadeh static int ceph_update_writeable_page(struct file *file, 13154af6b225SYehuda Sadeh loff_t pos, unsigned len, 13164af6b225SYehuda Sadeh struct page *page) 13171d3576fdSSage Weil { 1318496ad9aaSAl Viro struct inode *inode = file_inode(file); 13191d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 132018d620f0SJeff Layton struct ceph_snap_context *snapc; 132109cbfeafSKirill A. Shutemov loff_t page_off = pos & PAGE_MASK; 132209cbfeafSKirill A. Shutemov int pos_in_page = pos & ~PAGE_MASK; 13231d3576fdSSage Weil int end_in_page = pos_in_page + len; 13241d3576fdSSage Weil loff_t i_size; 13251d3576fdSSage Weil int r; 13266c93df5dSYan, Zheng 13271d3576fdSSage Weil retry_locked: 1328d45156bfSJeff Layton snapc = ceph_find_incompatible(page); 132918d620f0SJeff Layton if (snapc) { 133018d620f0SJeff Layton if (IS_ERR(snapc)) { 133118d620f0SJeff Layton r = PTR_ERR(snapc); 133218d620f0SJeff Layton goto fail_unlock; 133318d620f0SJeff Layton } 13341d3576fdSSage Weil unlock_page(page); 13353c6f6b79SSage Weil ceph_queue_writeback(inode); 1336a78bbd4bSYan, Zheng r = wait_event_killable(ci->i_cap_wq, 13371d3576fdSSage Weil context_is_writeable_or_written(inode, snapc)); 13381d3576fdSSage Weil ceph_put_snap_context(snapc); 13394af6b225SYehuda Sadeh return -EAGAIN; 13401d3576fdSSage Weil } 13411d3576fdSSage Weil 13421d3576fdSSage Weil if (PageUptodate(page)) { 13431d3576fdSSage Weil dout(" page %p already uptodate\n", page); 13441d3576fdSSage Weil return 0; 13451d3576fdSSage Weil } 13461d3576fdSSage Weil 13471d3576fdSSage Weil /* full page? */ 134809cbfeafSKirill A. Shutemov if (pos_in_page == 0 && len == PAGE_SIZE) 13491d3576fdSSage Weil return 0; 13501d3576fdSSage Weil 13511d3576fdSSage Weil /* past end of file? */ 135299c88e69SYan, Zheng i_size = i_size_read(inode); 13531d3576fdSSage Weil 13541d3576fdSSage Weil if (page_off >= i_size || 13551d3576fdSSage Weil (pos_in_page == 0 && (pos+len) >= i_size && 135609cbfeafSKirill A. Shutemov end_in_page - pos_in_page != PAGE_SIZE)) { 13571d3576fdSSage Weil dout(" zeroing %p 0 - %d and %d - %d\n", 135809cbfeafSKirill A. Shutemov page, pos_in_page, end_in_page, (int)PAGE_SIZE); 13591d3576fdSSage Weil zero_user_segments(page, 13601d3576fdSSage Weil 0, pos_in_page, 136109cbfeafSKirill A. Shutemov end_in_page, PAGE_SIZE); 13621d3576fdSSage Weil return 0; 13631d3576fdSSage Weil } 13641d3576fdSSage Weil 13651d3576fdSSage Weil /* we need to read it. */ 1366dd2bc473SYan, Zheng r = ceph_do_readpage(file, page); 1367dd2bc473SYan, Zheng if (r < 0) { 1368dd2bc473SYan, Zheng if (r == -EINPROGRESS) 1369dd2bc473SYan, Zheng return -EAGAIN; 1370dd2bc473SYan, Zheng goto fail_unlock; 1371dd2bc473SYan, Zheng } 13721d3576fdSSage Weil goto retry_locked; 1373dd2bc473SYan, Zheng fail_unlock: 13741d3576fdSSage Weil unlock_page(page); 13751d3576fdSSage Weil return r; 13761d3576fdSSage Weil } 13771d3576fdSSage Weil 13781d3576fdSSage Weil /* 13794af6b225SYehuda Sadeh * We are only allowed to write into/dirty the page if the page is 13804af6b225SYehuda Sadeh * clean, or already dirty within the same snap context. 13814af6b225SYehuda Sadeh */ 13824af6b225SYehuda Sadeh static int ceph_write_begin(struct file *file, struct address_space *mapping, 13834af6b225SYehuda Sadeh loff_t pos, unsigned len, unsigned flags, 13844af6b225SYehuda Sadeh struct page **pagep, void **fsdata) 13854af6b225SYehuda Sadeh { 1386496ad9aaSAl Viro struct inode *inode = file_inode(file); 13874af6b225SYehuda Sadeh struct page *page; 138809cbfeafSKirill A. Shutemov pgoff_t index = pos >> PAGE_SHIFT; 13897971bd92SSage Weil int r; 13904af6b225SYehuda Sadeh 13914af6b225SYehuda Sadeh do { 13924af6b225SYehuda Sadeh /* get a page */ 13934af6b225SYehuda Sadeh page = grab_cache_page_write_begin(mapping, index, 0); 13947971bd92SSage Weil if (!page) 13957971bd92SSage Weil return -ENOMEM; 13964af6b225SYehuda Sadeh 13974af6b225SYehuda Sadeh dout("write_begin file %p inode %p page %p %d~%d\n", file, 13984af6b225SYehuda Sadeh inode, page, (int)pos, (int)len); 13994af6b225SYehuda Sadeh 14004af6b225SYehuda Sadeh r = ceph_update_writeable_page(file, pos, len, page); 1401c1d00b2dSTaesoo Kim if (r < 0) 140209cbfeafSKirill A. Shutemov put_page(page); 1403c1d00b2dSTaesoo Kim else 1404c1d00b2dSTaesoo Kim *pagep = page; 14054af6b225SYehuda Sadeh } while (r == -EAGAIN); 14064af6b225SYehuda Sadeh 14074af6b225SYehuda Sadeh return r; 14084af6b225SYehuda Sadeh } 14094af6b225SYehuda Sadeh 14104af6b225SYehuda Sadeh /* 14111d3576fdSSage Weil * we don't do anything in here that simple_write_end doesn't do 14125dda377cSYan, Zheng * except adjust dirty page accounting 14131d3576fdSSage Weil */ 14141d3576fdSSage Weil static int ceph_write_end(struct file *file, struct address_space *mapping, 14151d3576fdSSage Weil loff_t pos, unsigned len, unsigned copied, 14161d3576fdSSage Weil struct page *page, void *fsdata) 14171d3576fdSSage Weil { 1418496ad9aaSAl Viro struct inode *inode = file_inode(file); 1419efb0ca76SYan, Zheng bool check_cap = false; 14201d3576fdSSage Weil 14211d3576fdSSage Weil dout("write_end file %p inode %p page %p %d~%d (%d)\n", file, 14221d3576fdSSage Weil inode, page, (int)pos, (int)copied, (int)len); 14231d3576fdSSage Weil 14241d3576fdSSage Weil /* zero the stale part of the page if we did a short copy */ 1425b9de313cSAl Viro if (!PageUptodate(page)) { 1426b9de313cSAl Viro if (copied < len) { 1427b9de313cSAl Viro copied = 0; 1428b9de313cSAl Viro goto out; 1429b9de313cSAl Viro } 1430b9de313cSAl Viro SetPageUptodate(page); 1431b9de313cSAl Viro } 14321d3576fdSSage Weil 14331d3576fdSSage Weil /* did file size increase? */ 143499c88e69SYan, Zheng if (pos+copied > i_size_read(inode)) 14351d3576fdSSage Weil check_cap = ceph_inode_set_size(inode, pos+copied); 14361d3576fdSSage Weil 14371d3576fdSSage Weil set_page_dirty(page); 14381d3576fdSSage Weil 1439b9de313cSAl Viro out: 14401d3576fdSSage Weil unlock_page(page); 144109cbfeafSKirill A. Shutemov put_page(page); 14421d3576fdSSage Weil 14431d3576fdSSage Weil if (check_cap) 14441d3576fdSSage Weil ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY, NULL); 14451d3576fdSSage Weil 14461d3576fdSSage Weil return copied; 14471d3576fdSSage Weil } 14481d3576fdSSage Weil 14491d3576fdSSage Weil /* 14501d3576fdSSage Weil * we set .direct_IO to indicate direct io is supported, but since we 14511d3576fdSSage Weil * intercept O_DIRECT reads and writes early, this function should 14521d3576fdSSage Weil * never get called. 14531d3576fdSSage Weil */ 1454c8b8e32dSChristoph Hellwig static ssize_t ceph_direct_io(struct kiocb *iocb, struct iov_iter *iter) 14551d3576fdSSage Weil { 14561d3576fdSSage Weil WARN_ON(1); 14571d3576fdSSage Weil return -EINVAL; 14581d3576fdSSage Weil } 14591d3576fdSSage Weil 14601d3576fdSSage Weil const struct address_space_operations ceph_aops = { 14611d3576fdSSage Weil .readpage = ceph_readpage, 14621d3576fdSSage Weil .readpages = ceph_readpages, 14631d3576fdSSage Weil .writepage = ceph_writepage, 14641d3576fdSSage Weil .writepages = ceph_writepages_start, 14651d3576fdSSage Weil .write_begin = ceph_write_begin, 14661d3576fdSSage Weil .write_end = ceph_write_end, 14671d3576fdSSage Weil .set_page_dirty = ceph_set_page_dirty, 14681d3576fdSSage Weil .invalidatepage = ceph_invalidatepage, 14691d3576fdSSage Weil .releasepage = ceph_releasepage, 14701d3576fdSSage Weil .direct_IO = ceph_direct_io, 14711d3576fdSSage Weil }; 14721d3576fdSSage Weil 14734f7e89f6SYan, Zheng static void ceph_block_sigs(sigset_t *oldset) 14744f7e89f6SYan, Zheng { 14754f7e89f6SYan, Zheng sigset_t mask; 14764f7e89f6SYan, Zheng siginitsetinv(&mask, sigmask(SIGKILL)); 14774f7e89f6SYan, Zheng sigprocmask(SIG_BLOCK, &mask, oldset); 14784f7e89f6SYan, Zheng } 14794f7e89f6SYan, Zheng 14804f7e89f6SYan, Zheng static void ceph_restore_sigs(sigset_t *oldset) 14814f7e89f6SYan, Zheng { 14824f7e89f6SYan, Zheng sigprocmask(SIG_SETMASK, oldset, NULL); 14834f7e89f6SYan, Zheng } 14841d3576fdSSage Weil 14851d3576fdSSage Weil /* 14861d3576fdSSage Weil * vm ops 14871d3576fdSSage Weil */ 148824499847SSouptick Joarder static vm_fault_t ceph_filemap_fault(struct vm_fault *vmf) 148961f68816SYan, Zheng { 149011bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 149161f68816SYan, Zheng struct inode *inode = file_inode(vma->vm_file); 149261f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 149361f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 14943738daa6SYan, Zheng struct page *pinned_page = NULL; 149509cbfeafSKirill A. Shutemov loff_t off = vmf->pgoff << PAGE_SHIFT; 149624499847SSouptick Joarder int want, got, err; 14974f7e89f6SYan, Zheng sigset_t oldset; 149824499847SSouptick Joarder vm_fault_t ret = VM_FAULT_SIGBUS; 14994f7e89f6SYan, Zheng 15004f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 150161f68816SYan, Zheng 150261f68816SYan, Zheng dout("filemap_fault %p %llx.%llx %llu~%zd trying to get caps\n", 150309cbfeafSKirill A. Shutemov inode, ceph_vinop(inode), off, (size_t)PAGE_SIZE); 150461f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 150561f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; 150661f68816SYan, Zheng else 150761f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE; 15084f7e89f6SYan, Zheng 150961f68816SYan, Zheng got = 0; 15105e3ded1bSYan, Zheng err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_RD, want, -1, 15115e3ded1bSYan, Zheng &got, &pinned_page); 151224499847SSouptick Joarder if (err < 0) 15134f7e89f6SYan, Zheng goto out_restore; 15146ce026e4SYan, Zheng 151561f68816SYan, Zheng dout("filemap_fault %p %llu~%zd got cap refs on %s\n", 151609cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ceph_cap_string(got)); 151761f68816SYan, Zheng 151883701246SYan, Zheng if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || 15192b1ac852SYan, Zheng ci->i_inline_version == CEPH_INLINE_NONE) { 15205d988308SYan, Zheng CEPH_DEFINE_RW_CONTEXT(rw_ctx, got); 15215d988308SYan, Zheng ceph_add_rw_context(fi, &rw_ctx); 152211bac800SDave Jiang ret = filemap_fault(vmf); 15235d988308SYan, Zheng ceph_del_rw_context(fi, &rw_ctx); 152424499847SSouptick Joarder dout("filemap_fault %p %llu~%zd drop cap refs %s ret %x\n", 152524499847SSouptick Joarder inode, off, (size_t)PAGE_SIZE, 152624499847SSouptick Joarder ceph_cap_string(got), ret); 15272b1ac852SYan, Zheng } else 152824499847SSouptick Joarder err = -EAGAIN; 152961f68816SYan, Zheng 15303738daa6SYan, Zheng if (pinned_page) 153109cbfeafSKirill A. Shutemov put_page(pinned_page); 153261f68816SYan, Zheng ceph_put_cap_refs(ci, got); 153361f68816SYan, Zheng 153424499847SSouptick Joarder if (err != -EAGAIN) 15354f7e89f6SYan, Zheng goto out_restore; 153683701246SYan, Zheng 153783701246SYan, Zheng /* read inline data */ 153809cbfeafSKirill A. Shutemov if (off >= PAGE_SIZE) { 153983701246SYan, Zheng /* does not support inline data > PAGE_SIZE */ 154083701246SYan, Zheng ret = VM_FAULT_SIGBUS; 154183701246SYan, Zheng } else { 154283701246SYan, Zheng struct address_space *mapping = inode->i_mapping; 154383701246SYan, Zheng struct page *page = find_or_create_page(mapping, 0, 1544c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1545c62d2555SMichal Hocko ~__GFP_FS)); 154683701246SYan, Zheng if (!page) { 154783701246SYan, Zheng ret = VM_FAULT_OOM; 15484f7e89f6SYan, Zheng goto out_inline; 154983701246SYan, Zheng } 155024499847SSouptick Joarder err = __ceph_do_getattr(inode, page, 155183701246SYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 155224499847SSouptick Joarder if (err < 0 || off >= i_size_read(inode)) { 155383701246SYan, Zheng unlock_page(page); 155409cbfeafSKirill A. Shutemov put_page(page); 1555c64a2b05SSouptick Joarder ret = vmf_error(err); 15564f7e89f6SYan, Zheng goto out_inline; 155783701246SYan, Zheng } 155824499847SSouptick Joarder if (err < PAGE_SIZE) 155924499847SSouptick Joarder zero_user_segment(page, err, PAGE_SIZE); 156083701246SYan, Zheng else 156183701246SYan, Zheng flush_dcache_page(page); 156283701246SYan, Zheng SetPageUptodate(page); 156383701246SYan, Zheng vmf->page = page; 156483701246SYan, Zheng ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; 15654f7e89f6SYan, Zheng out_inline: 156624499847SSouptick Joarder dout("filemap_fault %p %llu~%zd read inline data ret %x\n", 156709cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ret); 15684f7e89f6SYan, Zheng } 15694f7e89f6SYan, Zheng out_restore: 15704f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 157124499847SSouptick Joarder if (err < 0) 157224499847SSouptick Joarder ret = vmf_error(err); 15736ce026e4SYan, Zheng 157461f68816SYan, Zheng return ret; 157561f68816SYan, Zheng } 15761d3576fdSSage Weil 15771d3576fdSSage Weil /* 15781d3576fdSSage Weil * Reuse write_begin here for simplicity. 15791d3576fdSSage Weil */ 158024499847SSouptick Joarder static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) 15811d3576fdSSage Weil { 158211bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 1583496ad9aaSAl Viro struct inode *inode = file_inode(vma->vm_file); 158461f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 158561f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 1586f66fd9f0SYan, Zheng struct ceph_cap_flush *prealloc_cf; 158761f68816SYan, Zheng struct page *page = vmf->page; 15886285bc23SAlex Elder loff_t off = page_offset(page); 158961f68816SYan, Zheng loff_t size = i_size_read(inode); 159061f68816SYan, Zheng size_t len; 159124499847SSouptick Joarder int want, got, err; 15924f7e89f6SYan, Zheng sigset_t oldset; 159324499847SSouptick Joarder vm_fault_t ret = VM_FAULT_SIGBUS; 15941d3576fdSSage Weil 1595f66fd9f0SYan, Zheng prealloc_cf = ceph_alloc_cap_flush(); 1596f66fd9f0SYan, Zheng if (!prealloc_cf) 15976ce026e4SYan, Zheng return VM_FAULT_OOM; 1598f66fd9f0SYan, Zheng 1599249c1df5SJeff Layton sb_start_pagefault(inode->i_sb); 16004f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 16011d3576fdSSage Weil 160228127bddSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 160328127bddSYan, Zheng struct page *locked_page = NULL; 160428127bddSYan, Zheng if (off == 0) { 160528127bddSYan, Zheng lock_page(page); 160628127bddSYan, Zheng locked_page = page; 160728127bddSYan, Zheng } 160824499847SSouptick Joarder err = ceph_uninline_data(vma->vm_file, locked_page); 160928127bddSYan, Zheng if (locked_page) 161028127bddSYan, Zheng unlock_page(locked_page); 161124499847SSouptick Joarder if (err < 0) 1612f66fd9f0SYan, Zheng goto out_free; 1613f66fd9f0SYan, Zheng } 161428127bddSYan, Zheng 161509cbfeafSKirill A. Shutemov if (off + PAGE_SIZE <= size) 161609cbfeafSKirill A. Shutemov len = PAGE_SIZE; 16171d3576fdSSage Weil else 161809cbfeafSKirill A. Shutemov len = size & ~PAGE_MASK; 16191d3576fdSSage Weil 162061f68816SYan, Zheng dout("page_mkwrite %p %llx.%llx %llu~%zd getting caps i_size %llu\n", 162161f68816SYan, Zheng inode, ceph_vinop(inode), off, len, size); 162261f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 162361f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; 162461f68816SYan, Zheng else 162561f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER; 16264f7e89f6SYan, Zheng 162761f68816SYan, Zheng got = 0; 16285e3ded1bSYan, Zheng err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_WR, want, off + len, 16293738daa6SYan, Zheng &got, NULL); 163024499847SSouptick Joarder if (err < 0) 1631f66fd9f0SYan, Zheng goto out_free; 16326ce026e4SYan, Zheng 163361f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd got cap refs on %s\n", 163461f68816SYan, Zheng inode, off, len, ceph_cap_string(got)); 163561f68816SYan, Zheng 163661f68816SYan, Zheng /* Update time before taking page lock */ 163761f68816SYan, Zheng file_update_time(vma->vm_file); 16385c308356SJeff Layton inode_inc_iversion_raw(inode); 16394af6b225SYehuda Sadeh 1640f0b33df5SYan, Zheng do { 1641d45156bfSJeff Layton struct ceph_snap_context *snapc; 1642d45156bfSJeff Layton 16434af6b225SYehuda Sadeh lock_page(page); 16444af6b225SYehuda Sadeh 1645cb03c143SAndreas Gruenbacher if (page_mkwrite_check_truncate(page, inode) < 0) { 1646f9cac5acSYan, Zheng unlock_page(page); 16476ce026e4SYan, Zheng ret = VM_FAULT_NOPAGE; 1648f0b33df5SYan, Zheng break; 1649f9cac5acSYan, Zheng } 16504af6b225SYehuda Sadeh 1651d45156bfSJeff Layton snapc = ceph_find_incompatible(page); 1652d45156bfSJeff Layton if (!snapc) { 16534af6b225SYehuda Sadeh /* success. we'll keep the page locked. */ 16541d3576fdSSage Weil set_page_dirty(page); 16551d3576fdSSage Weil ret = VM_FAULT_LOCKED; 1656d45156bfSJeff Layton break; 16571d3576fdSSage Weil } 1658d45156bfSJeff Layton 1659d45156bfSJeff Layton unlock_page(page); 1660d45156bfSJeff Layton 1661d45156bfSJeff Layton if (IS_ERR(snapc)) { 1662d45156bfSJeff Layton ret = VM_FAULT_SIGBUS; 1663d45156bfSJeff Layton break; 1664d45156bfSJeff Layton } 1665d45156bfSJeff Layton 1666d45156bfSJeff Layton ceph_queue_writeback(inode); 1667d45156bfSJeff Layton err = wait_event_killable(ci->i_cap_wq, 1668d45156bfSJeff Layton context_is_writeable_or_written(inode, snapc)); 1669d45156bfSJeff Layton ceph_put_snap_context(snapc); 1670d45156bfSJeff Layton } while (err == 0); 1671f0b33df5SYan, Zheng 167228127bddSYan, Zheng if (ret == VM_FAULT_LOCKED || 167328127bddSYan, Zheng ci->i_inline_version != CEPH_INLINE_NONE) { 167461f68816SYan, Zheng int dirty; 167561f68816SYan, Zheng spin_lock(&ci->i_ceph_lock); 167628127bddSYan, Zheng ci->i_inline_version = CEPH_INLINE_NONE; 1677f66fd9f0SYan, Zheng dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, 1678f66fd9f0SYan, Zheng &prealloc_cf); 167961f68816SYan, Zheng spin_unlock(&ci->i_ceph_lock); 168061f68816SYan, Zheng if (dirty) 168161f68816SYan, Zheng __mark_inode_dirty(inode, dirty); 168261f68816SYan, Zheng } 168361f68816SYan, Zheng 168424499847SSouptick Joarder dout("page_mkwrite %p %llu~%zd dropping cap refs on %s ret %x\n", 168561f68816SYan, Zheng inode, off, len, ceph_cap_string(got), ret); 168661f68816SYan, Zheng ceph_put_cap_refs(ci, got); 1687f66fd9f0SYan, Zheng out_free: 16884f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 1689249c1df5SJeff Layton sb_end_pagefault(inode->i_sb); 1690f66fd9f0SYan, Zheng ceph_free_cap_flush(prealloc_cf); 169124499847SSouptick Joarder if (err < 0) 169224499847SSouptick Joarder ret = vmf_error(err); 16931d3576fdSSage Weil return ret; 16941d3576fdSSage Weil } 16951d3576fdSSage Weil 169631c542a1SYan, Zheng void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, 169731c542a1SYan, Zheng char *data, size_t len) 169831c542a1SYan, Zheng { 169931c542a1SYan, Zheng struct address_space *mapping = inode->i_mapping; 170031c542a1SYan, Zheng struct page *page; 170131c542a1SYan, Zheng 170231c542a1SYan, Zheng if (locked_page) { 170331c542a1SYan, Zheng page = locked_page; 170431c542a1SYan, Zheng } else { 170531c542a1SYan, Zheng if (i_size_read(inode) == 0) 170631c542a1SYan, Zheng return; 170731c542a1SYan, Zheng page = find_or_create_page(mapping, 0, 1708c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1709c62d2555SMichal Hocko ~__GFP_FS)); 171031c542a1SYan, Zheng if (!page) 171131c542a1SYan, Zheng return; 171231c542a1SYan, Zheng if (PageUptodate(page)) { 171331c542a1SYan, Zheng unlock_page(page); 171409cbfeafSKirill A. Shutemov put_page(page); 171531c542a1SYan, Zheng return; 171631c542a1SYan, Zheng } 171731c542a1SYan, Zheng } 171831c542a1SYan, Zheng 17190668ff52SIlya Dryomov dout("fill_inline_data %p %llx.%llx len %zu locked_page %p\n", 172031c542a1SYan, Zheng inode, ceph_vinop(inode), len, locked_page); 172131c542a1SYan, Zheng 172231c542a1SYan, Zheng if (len > 0) { 172331c542a1SYan, Zheng void *kaddr = kmap_atomic(page); 172431c542a1SYan, Zheng memcpy(kaddr, data, len); 172531c542a1SYan, Zheng kunmap_atomic(kaddr); 172631c542a1SYan, Zheng } 172731c542a1SYan, Zheng 172831c542a1SYan, Zheng if (page != locked_page) { 172909cbfeafSKirill A. Shutemov if (len < PAGE_SIZE) 173009cbfeafSKirill A. Shutemov zero_user_segment(page, len, PAGE_SIZE); 173131c542a1SYan, Zheng else 173231c542a1SYan, Zheng flush_dcache_page(page); 173331c542a1SYan, Zheng 173431c542a1SYan, Zheng SetPageUptodate(page); 173531c542a1SYan, Zheng unlock_page(page); 173609cbfeafSKirill A. Shutemov put_page(page); 173731c542a1SYan, Zheng } 173831c542a1SYan, Zheng } 173931c542a1SYan, Zheng 174028127bddSYan, Zheng int ceph_uninline_data(struct file *filp, struct page *locked_page) 174128127bddSYan, Zheng { 174228127bddSYan, Zheng struct inode *inode = file_inode(filp); 174328127bddSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 174428127bddSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 174528127bddSYan, Zheng struct ceph_osd_request *req; 174628127bddSYan, Zheng struct page *page = NULL; 174728127bddSYan, Zheng u64 len, inline_version; 174828127bddSYan, Zheng int err = 0; 174928127bddSYan, Zheng bool from_pagecache = false; 175028127bddSYan, Zheng 175128127bddSYan, Zheng spin_lock(&ci->i_ceph_lock); 175228127bddSYan, Zheng inline_version = ci->i_inline_version; 175328127bddSYan, Zheng spin_unlock(&ci->i_ceph_lock); 175428127bddSYan, Zheng 175528127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu\n", 175628127bddSYan, Zheng inode, ceph_vinop(inode), inline_version); 175728127bddSYan, Zheng 175828127bddSYan, Zheng if (inline_version == 1 || /* initial version, no data */ 175928127bddSYan, Zheng inline_version == CEPH_INLINE_NONE) 176028127bddSYan, Zheng goto out; 176128127bddSYan, Zheng 176228127bddSYan, Zheng if (locked_page) { 176328127bddSYan, Zheng page = locked_page; 176428127bddSYan, Zheng WARN_ON(!PageUptodate(page)); 176528127bddSYan, Zheng } else if (ceph_caps_issued(ci) & 176628127bddSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) { 176728127bddSYan, Zheng page = find_get_page(inode->i_mapping, 0); 176828127bddSYan, Zheng if (page) { 176928127bddSYan, Zheng if (PageUptodate(page)) { 177028127bddSYan, Zheng from_pagecache = true; 177128127bddSYan, Zheng lock_page(page); 177228127bddSYan, Zheng } else { 177309cbfeafSKirill A. Shutemov put_page(page); 177428127bddSYan, Zheng page = NULL; 177528127bddSYan, Zheng } 177628127bddSYan, Zheng } 177728127bddSYan, Zheng } 177828127bddSYan, Zheng 177928127bddSYan, Zheng if (page) { 178028127bddSYan, Zheng len = i_size_read(inode); 178109cbfeafSKirill A. Shutemov if (len > PAGE_SIZE) 178209cbfeafSKirill A. Shutemov len = PAGE_SIZE; 178328127bddSYan, Zheng } else { 178428127bddSYan, Zheng page = __page_cache_alloc(GFP_NOFS); 178528127bddSYan, Zheng if (!page) { 178628127bddSYan, Zheng err = -ENOMEM; 178728127bddSYan, Zheng goto out; 178828127bddSYan, Zheng } 178928127bddSYan, Zheng err = __ceph_do_getattr(inode, page, 179028127bddSYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 179128127bddSYan, Zheng if (err < 0) { 179228127bddSYan, Zheng /* no inline data */ 179328127bddSYan, Zheng if (err == -ENODATA) 179428127bddSYan, Zheng err = 0; 179528127bddSYan, Zheng goto out; 179628127bddSYan, Zheng } 179728127bddSYan, Zheng len = err; 179828127bddSYan, Zheng } 179928127bddSYan, Zheng 180028127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 180128127bddSYan, Zheng ceph_vino(inode), 0, &len, 0, 1, 180254ea0046SIlya Dryomov CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, 180334b759b4SIlya Dryomov NULL, 0, 0, false); 180428127bddSYan, Zheng if (IS_ERR(req)) { 180528127bddSYan, Zheng err = PTR_ERR(req); 180628127bddSYan, Zheng goto out; 180728127bddSYan, Zheng } 180828127bddSYan, Zheng 1809fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 181028127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 181128127bddSYan, Zheng if (!err) 181228127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 181328127bddSYan, Zheng ceph_osdc_put_request(req); 181428127bddSYan, Zheng if (err < 0) 181528127bddSYan, Zheng goto out; 181628127bddSYan, Zheng 181728127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 181828127bddSYan, Zheng ceph_vino(inode), 0, &len, 1, 3, 181954ea0046SIlya Dryomov CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 182034b759b4SIlya Dryomov NULL, ci->i_truncate_seq, 182134b759b4SIlya Dryomov ci->i_truncate_size, false); 182228127bddSYan, Zheng if (IS_ERR(req)) { 182328127bddSYan, Zheng err = PTR_ERR(req); 182428127bddSYan, Zheng goto out; 182528127bddSYan, Zheng } 182628127bddSYan, Zheng 182728127bddSYan, Zheng osd_req_op_extent_osd_data_pages(req, 1, &page, len, 0, false, false); 182828127bddSYan, Zheng 1829ec137c10SYan, Zheng { 1830ec137c10SYan, Zheng __le64 xattr_buf = cpu_to_le64(inline_version); 183128127bddSYan, Zheng err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, 1832ec137c10SYan, Zheng "inline_version", &xattr_buf, 1833ec137c10SYan, Zheng sizeof(xattr_buf), 183428127bddSYan, Zheng CEPH_OSD_CMPXATTR_OP_GT, 183528127bddSYan, Zheng CEPH_OSD_CMPXATTR_MODE_U64); 183628127bddSYan, Zheng if (err) 183728127bddSYan, Zheng goto out_put; 1838ec137c10SYan, Zheng } 183928127bddSYan, Zheng 1840ec137c10SYan, Zheng { 1841ec137c10SYan, Zheng char xattr_buf[32]; 1842ec137c10SYan, Zheng int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), 1843ec137c10SYan, Zheng "%llu", inline_version); 184428127bddSYan, Zheng err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, 1845ec137c10SYan, Zheng "inline_version", 1846ec137c10SYan, Zheng xattr_buf, xattr_len, 0, 0); 184728127bddSYan, Zheng if (err) 184828127bddSYan, Zheng goto out_put; 1849ec137c10SYan, Zheng } 185028127bddSYan, Zheng 1851fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 185228127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 185328127bddSYan, Zheng if (!err) 185428127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 185597e27aaaSXiubo Li 185697e27aaaSXiubo Li ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 185797e27aaaSXiubo Li req->r_end_latency, err); 185897e27aaaSXiubo Li 185928127bddSYan, Zheng out_put: 186028127bddSYan, Zheng ceph_osdc_put_request(req); 186128127bddSYan, Zheng if (err == -ECANCELED) 186228127bddSYan, Zheng err = 0; 186328127bddSYan, Zheng out: 186428127bddSYan, Zheng if (page && page != locked_page) { 186528127bddSYan, Zheng if (from_pagecache) { 186628127bddSYan, Zheng unlock_page(page); 186709cbfeafSKirill A. Shutemov put_page(page); 186828127bddSYan, Zheng } else 186928127bddSYan, Zheng __free_pages(page, 0); 187028127bddSYan, Zheng } 187128127bddSYan, Zheng 187228127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu = %d\n", 187328127bddSYan, Zheng inode, ceph_vinop(inode), inline_version, err); 187428127bddSYan, Zheng return err; 187528127bddSYan, Zheng } 187628127bddSYan, Zheng 18777cbea8dcSKirill A. Shutemov static const struct vm_operations_struct ceph_vmops = { 187861f68816SYan, Zheng .fault = ceph_filemap_fault, 18791d3576fdSSage Weil .page_mkwrite = ceph_page_mkwrite, 18801d3576fdSSage Weil }; 18811d3576fdSSage Weil 18821d3576fdSSage Weil int ceph_mmap(struct file *file, struct vm_area_struct *vma) 18831d3576fdSSage Weil { 18841d3576fdSSage Weil struct address_space *mapping = file->f_mapping; 18851d3576fdSSage Weil 18861d3576fdSSage Weil if (!mapping->a_ops->readpage) 18871d3576fdSSage Weil return -ENOEXEC; 18881d3576fdSSage Weil file_accessed(file); 18891d3576fdSSage Weil vma->vm_ops = &ceph_vmops; 18901d3576fdSSage Weil return 0; 18911d3576fdSSage Weil } 189210183a69SYan, Zheng 189310183a69SYan, Zheng enum { 189410183a69SYan, Zheng POOL_READ = 1, 189510183a69SYan, Zheng POOL_WRITE = 2, 189610183a69SYan, Zheng }; 189710183a69SYan, Zheng 1898779fe0fbSYan, Zheng static int __ceph_pool_perm_get(struct ceph_inode_info *ci, 1899779fe0fbSYan, Zheng s64 pool, struct ceph_string *pool_ns) 190010183a69SYan, Zheng { 190110183a69SYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(&ci->vfs_inode); 190210183a69SYan, Zheng struct ceph_mds_client *mdsc = fsc->mdsc; 190310183a69SYan, Zheng struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; 190410183a69SYan, Zheng struct rb_node **p, *parent; 190510183a69SYan, Zheng struct ceph_pool_perm *perm; 190610183a69SYan, Zheng struct page **pages; 1907779fe0fbSYan, Zheng size_t pool_ns_len; 190810183a69SYan, Zheng int err = 0, err2 = 0, have = 0; 190910183a69SYan, Zheng 191010183a69SYan, Zheng down_read(&mdsc->pool_perm_rwsem); 191110183a69SYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 191210183a69SYan, Zheng while (*p) { 191310183a69SYan, Zheng perm = rb_entry(*p, struct ceph_pool_perm, node); 191410183a69SYan, Zheng if (pool < perm->pool) 191510183a69SYan, Zheng p = &(*p)->rb_left; 191610183a69SYan, Zheng else if (pool > perm->pool) 191710183a69SYan, Zheng p = &(*p)->rb_right; 191810183a69SYan, Zheng else { 1919779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1920779fe0fbSYan, Zheng perm->pool_ns, 1921779fe0fbSYan, Zheng perm->pool_ns_len); 1922779fe0fbSYan, Zheng if (ret < 0) 1923779fe0fbSYan, Zheng p = &(*p)->rb_left; 1924779fe0fbSYan, Zheng else if (ret > 0) 1925779fe0fbSYan, Zheng p = &(*p)->rb_right; 1926779fe0fbSYan, Zheng else { 192710183a69SYan, Zheng have = perm->perm; 192810183a69SYan, Zheng break; 192910183a69SYan, Zheng } 193010183a69SYan, Zheng } 1931779fe0fbSYan, Zheng } 193210183a69SYan, Zheng up_read(&mdsc->pool_perm_rwsem); 193310183a69SYan, Zheng if (*p) 193410183a69SYan, Zheng goto out; 193510183a69SYan, Zheng 1936779fe0fbSYan, Zheng if (pool_ns) 1937779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s no perm cached\n", 1938779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str); 1939779fe0fbSYan, Zheng else 19407627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld no perm cached\n", pool); 194110183a69SYan, Zheng 194210183a69SYan, Zheng down_write(&mdsc->pool_perm_rwsem); 1943779fe0fbSYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 194410183a69SYan, Zheng parent = NULL; 194510183a69SYan, Zheng while (*p) { 194610183a69SYan, Zheng parent = *p; 194710183a69SYan, Zheng perm = rb_entry(parent, struct ceph_pool_perm, node); 194810183a69SYan, Zheng if (pool < perm->pool) 194910183a69SYan, Zheng p = &(*p)->rb_left; 195010183a69SYan, Zheng else if (pool > perm->pool) 195110183a69SYan, Zheng p = &(*p)->rb_right; 195210183a69SYan, Zheng else { 1953779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1954779fe0fbSYan, Zheng perm->pool_ns, 1955779fe0fbSYan, Zheng perm->pool_ns_len); 1956779fe0fbSYan, Zheng if (ret < 0) 1957779fe0fbSYan, Zheng p = &(*p)->rb_left; 1958779fe0fbSYan, Zheng else if (ret > 0) 1959779fe0fbSYan, Zheng p = &(*p)->rb_right; 1960779fe0fbSYan, Zheng else { 196110183a69SYan, Zheng have = perm->perm; 196210183a69SYan, Zheng break; 196310183a69SYan, Zheng } 196410183a69SYan, Zheng } 1965779fe0fbSYan, Zheng } 196610183a69SYan, Zheng if (*p) { 196710183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 196810183a69SYan, Zheng goto out; 196910183a69SYan, Zheng } 197010183a69SYan, Zheng 197134b759b4SIlya Dryomov rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 197210183a69SYan, Zheng 1, false, GFP_NOFS); 197310183a69SYan, Zheng if (!rd_req) { 197410183a69SYan, Zheng err = -ENOMEM; 197510183a69SYan, Zheng goto out_unlock; 197610183a69SYan, Zheng } 197710183a69SYan, Zheng 197810183a69SYan, Zheng rd_req->r_flags = CEPH_OSD_FLAG_READ; 197910183a69SYan, Zheng osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); 198010183a69SYan, Zheng rd_req->r_base_oloc.pool = pool; 1981779fe0fbSYan, Zheng if (pool_ns) 1982779fe0fbSYan, Zheng rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns); 1983d30291b9SIlya Dryomov ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ci->i_vino.ino); 198410183a69SYan, Zheng 198513d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(rd_req, GFP_NOFS); 198613d1ad16SIlya Dryomov if (err) 198713d1ad16SIlya Dryomov goto out_unlock; 198810183a69SYan, Zheng 198934b759b4SIlya Dryomov wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 199010183a69SYan, Zheng 1, false, GFP_NOFS); 199110183a69SYan, Zheng if (!wr_req) { 199210183a69SYan, Zheng err = -ENOMEM; 199310183a69SYan, Zheng goto out_unlock; 199410183a69SYan, Zheng } 199510183a69SYan, Zheng 199654ea0046SIlya Dryomov wr_req->r_flags = CEPH_OSD_FLAG_WRITE; 199710183a69SYan, Zheng osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); 199863244fa1SIlya Dryomov ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc); 1999d30291b9SIlya Dryomov ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid); 200010183a69SYan, Zheng 200113d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(wr_req, GFP_NOFS); 200213d1ad16SIlya Dryomov if (err) 200313d1ad16SIlya Dryomov goto out_unlock; 200410183a69SYan, Zheng 200510183a69SYan, Zheng /* one page should be large enough for STAT data */ 200610183a69SYan, Zheng pages = ceph_alloc_page_vector(1, GFP_KERNEL); 200710183a69SYan, Zheng if (IS_ERR(pages)) { 200810183a69SYan, Zheng err = PTR_ERR(pages); 200910183a69SYan, Zheng goto out_unlock; 201010183a69SYan, Zheng } 201110183a69SYan, Zheng 201210183a69SYan, Zheng osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, 201310183a69SYan, Zheng 0, false, true); 201410183a69SYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false); 201510183a69SYan, Zheng 2016fac02ddfSArnd Bergmann wr_req->r_mtime = ci->vfs_inode.i_mtime; 201710183a69SYan, Zheng err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false); 201810183a69SYan, Zheng 201910183a69SYan, Zheng if (!err) 202010183a69SYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); 202110183a69SYan, Zheng if (!err2) 202210183a69SYan, Zheng err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); 202310183a69SYan, Zheng 202410183a69SYan, Zheng if (err >= 0 || err == -ENOENT) 202510183a69SYan, Zheng have |= POOL_READ; 2026131d7eb4SYan, Zheng else if (err != -EPERM) { 20270b98acd6SIlya Dryomov if (err == -EBLOCKLISTED) 20280b98acd6SIlya Dryomov fsc->blocklisted = true; 202910183a69SYan, Zheng goto out_unlock; 2030131d7eb4SYan, Zheng } 203110183a69SYan, Zheng 203210183a69SYan, Zheng if (err2 == 0 || err2 == -EEXIST) 203310183a69SYan, Zheng have |= POOL_WRITE; 203410183a69SYan, Zheng else if (err2 != -EPERM) { 20350b98acd6SIlya Dryomov if (err2 == -EBLOCKLISTED) 20360b98acd6SIlya Dryomov fsc->blocklisted = true; 203710183a69SYan, Zheng err = err2; 203810183a69SYan, Zheng goto out_unlock; 203910183a69SYan, Zheng } 204010183a69SYan, Zheng 2041779fe0fbSYan, Zheng pool_ns_len = pool_ns ? pool_ns->len : 0; 2042779fe0fbSYan, Zheng perm = kmalloc(sizeof(*perm) + pool_ns_len + 1, GFP_NOFS); 204310183a69SYan, Zheng if (!perm) { 204410183a69SYan, Zheng err = -ENOMEM; 204510183a69SYan, Zheng goto out_unlock; 204610183a69SYan, Zheng } 204710183a69SYan, Zheng 204810183a69SYan, Zheng perm->pool = pool; 204910183a69SYan, Zheng perm->perm = have; 2050779fe0fbSYan, Zheng perm->pool_ns_len = pool_ns_len; 2051779fe0fbSYan, Zheng if (pool_ns_len > 0) 2052779fe0fbSYan, Zheng memcpy(perm->pool_ns, pool_ns->str, pool_ns_len); 2053779fe0fbSYan, Zheng perm->pool_ns[pool_ns_len] = 0; 2054779fe0fbSYan, Zheng 205510183a69SYan, Zheng rb_link_node(&perm->node, parent, p); 205610183a69SYan, Zheng rb_insert_color(&perm->node, &mdsc->pool_perm_tree); 205710183a69SYan, Zheng err = 0; 205810183a69SYan, Zheng out_unlock: 205910183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 206010183a69SYan, Zheng 206110183a69SYan, Zheng ceph_osdc_put_request(rd_req); 206210183a69SYan, Zheng ceph_osdc_put_request(wr_req); 206310183a69SYan, Zheng out: 206410183a69SYan, Zheng if (!err) 206510183a69SYan, Zheng err = have; 2066779fe0fbSYan, Zheng if (pool_ns) 2067779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s result = %d\n", 2068779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str, err); 2069779fe0fbSYan, Zheng else 20707627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld result = %d\n", pool, err); 207110183a69SYan, Zheng return err; 207210183a69SYan, Zheng } 207310183a69SYan, Zheng 20745e3ded1bSYan, Zheng int ceph_pool_perm_check(struct inode *inode, int need) 207510183a69SYan, Zheng { 20765e3ded1bSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 2077779fe0fbSYan, Zheng struct ceph_string *pool_ns; 20785e3ded1bSYan, Zheng s64 pool; 207910183a69SYan, Zheng int ret, flags; 208010183a69SYan, Zheng 208180e80fbbSYan, Zheng if (ci->i_vino.snap != CEPH_NOSNAP) { 208280e80fbbSYan, Zheng /* 208380e80fbbSYan, Zheng * Pool permission check needs to write to the first object. 208480e80fbbSYan, Zheng * But for snapshot, head of the first object may have alread 208580e80fbbSYan, Zheng * been deleted. Skip check to avoid creating orphan object. 208680e80fbbSYan, Zheng */ 208780e80fbbSYan, Zheng return 0; 208880e80fbbSYan, Zheng } 208980e80fbbSYan, Zheng 20905e3ded1bSYan, Zheng if (ceph_test_mount_opt(ceph_inode_to_client(inode), 209110183a69SYan, Zheng NOPOOLPERM)) 209210183a69SYan, Zheng return 0; 209310183a69SYan, Zheng 209410183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 209510183a69SYan, Zheng flags = ci->i_ceph_flags; 20967627151eSYan, Zheng pool = ci->i_layout.pool_id; 209710183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 209810183a69SYan, Zheng check: 209910183a69SYan, Zheng if (flags & CEPH_I_POOL_PERM) { 210010183a69SYan, Zheng if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { 21017627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no read perm\n", 210210183a69SYan, Zheng pool); 210310183a69SYan, Zheng return -EPERM; 210410183a69SYan, Zheng } 210510183a69SYan, Zheng if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { 21067627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no write perm\n", 210710183a69SYan, Zheng pool); 210810183a69SYan, Zheng return -EPERM; 210910183a69SYan, Zheng } 211010183a69SYan, Zheng return 0; 211110183a69SYan, Zheng } 211210183a69SYan, Zheng 2113779fe0fbSYan, Zheng pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); 2114779fe0fbSYan, Zheng ret = __ceph_pool_perm_get(ci, pool, pool_ns); 2115779fe0fbSYan, Zheng ceph_put_string(pool_ns); 211610183a69SYan, Zheng if (ret < 0) 211710183a69SYan, Zheng return ret; 211810183a69SYan, Zheng 211910183a69SYan, Zheng flags = CEPH_I_POOL_PERM; 212010183a69SYan, Zheng if (ret & POOL_READ) 212110183a69SYan, Zheng flags |= CEPH_I_POOL_RD; 212210183a69SYan, Zheng if (ret & POOL_WRITE) 212310183a69SYan, Zheng flags |= CEPH_I_POOL_WR; 212410183a69SYan, Zheng 212510183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 2126779fe0fbSYan, Zheng if (pool == ci->i_layout.pool_id && 2127779fe0fbSYan, Zheng pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { 2128779fe0fbSYan, Zheng ci->i_ceph_flags |= flags; 212910183a69SYan, Zheng } else { 21307627151eSYan, Zheng pool = ci->i_layout.pool_id; 213110183a69SYan, Zheng flags = ci->i_ceph_flags; 213210183a69SYan, Zheng } 213310183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 213410183a69SYan, Zheng goto check; 213510183a69SYan, Zheng } 213610183a69SYan, Zheng 213710183a69SYan, Zheng void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) 213810183a69SYan, Zheng { 213910183a69SYan, Zheng struct ceph_pool_perm *perm; 214010183a69SYan, Zheng struct rb_node *n; 214110183a69SYan, Zheng 214210183a69SYan, Zheng while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { 214310183a69SYan, Zheng n = rb_first(&mdsc->pool_perm_tree); 214410183a69SYan, Zheng perm = rb_entry(n, struct ceph_pool_perm, node); 214510183a69SYan, Zheng rb_erase(n, &mdsc->pool_perm_tree); 214610183a69SYan, Zheng kfree(perm); 214710183a69SYan, Zheng } 214810183a69SYan, Zheng } 2149