1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 23d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 31d3576fdSSage Weil 41d3576fdSSage Weil #include <linux/backing-dev.h> 51d3576fdSSage Weil #include <linux/fs.h> 61d3576fdSSage Weil #include <linux/mm.h> 71d3576fdSSage Weil #include <linux/pagemap.h> 81d3576fdSSage Weil #include <linux/writeback.h> /* generic_writepages */ 95a0e3ad6STejun Heo #include <linux/slab.h> 101d3576fdSSage Weil #include <linux/pagevec.h> 111d3576fdSSage Weil #include <linux/task_io_accounting_ops.h> 12f361bf4aSIngo Molnar #include <linux/signal.h> 135c308356SJeff Layton #include <linux/iversion.h> 1497e27aaaSXiubo Li #include <linux/ktime.h> 15f0702876SJeff Layton #include <linux/netfs.h> 161d3576fdSSage Weil 171d3576fdSSage Weil #include "super.h" 183d14c5d2SYehuda Sadeh #include "mds_client.h" 1999ccbd22SMilosz Tanski #include "cache.h" 2097e27aaaSXiubo Li #include "metric.h" 213d14c5d2SYehuda Sadeh #include <linux/ceph/osd_client.h> 2208c1ac50SIlya Dryomov #include <linux/ceph/striper.h> 231d3576fdSSage Weil 241d3576fdSSage Weil /* 251d3576fdSSage Weil * Ceph address space ops. 261d3576fdSSage Weil * 271d3576fdSSage Weil * There are a few funny things going on here. 281d3576fdSSage Weil * 291d3576fdSSage Weil * The page->private field is used to reference a struct 301d3576fdSSage Weil * ceph_snap_context for _every_ dirty page. This indicates which 311d3576fdSSage Weil * snapshot the page was logically dirtied in, and thus which snap 321d3576fdSSage Weil * context needs to be associated with the osd write during writeback. 331d3576fdSSage Weil * 341d3576fdSSage Weil * Similarly, struct ceph_inode_info maintains a set of counters to 3525985edcSLucas De Marchi * count dirty pages on the inode. In the absence of snapshots, 361d3576fdSSage Weil * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. 371d3576fdSSage Weil * 381d3576fdSSage Weil * When a snapshot is taken (that is, when the client receives 391d3576fdSSage Weil * notification that a snapshot was taken), each inode with caps and 401d3576fdSSage Weil * with dirty pages (dirty pages implies there is a cap) gets a new 411d3576fdSSage Weil * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending 421d3576fdSSage Weil * order, new snaps go to the tail). The i_wrbuffer_ref_head count is 431d3576fdSSage Weil * moved to capsnap->dirty. (Unless a sync write is currently in 441d3576fdSSage Weil * progress. In that case, the capsnap is said to be "pending", new 451d3576fdSSage Weil * writes cannot start, and the capsnap isn't "finalized" until the 461d3576fdSSage Weil * write completes (or fails) and a final size/mtime for the inode for 471d3576fdSSage Weil * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. 481d3576fdSSage Weil * 491d3576fdSSage Weil * On writeback, we must submit writes to the osd IN SNAP ORDER. So, 501d3576fdSSage Weil * we look for the first capsnap in i_cap_snaps and write out pages in 511d3576fdSSage Weil * that snap context _only_. Then we move on to the next capsnap, 521d3576fdSSage Weil * eventually reaching the "live" or "head" context (i.e., pages that 531d3576fdSSage Weil * are not yet snapped) and are writing the most recently dirtied 541d3576fdSSage Weil * pages. 551d3576fdSSage Weil * 561d3576fdSSage Weil * Invalidate and so forth must take care to ensure the dirty page 571d3576fdSSage Weil * accounting is preserved. 581d3576fdSSage Weil */ 591d3576fdSSage Weil 602baba250SYehuda Sadeh #define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) 612baba250SYehuda Sadeh #define CONGESTION_OFF_THRESH(congestion_kb) \ 622baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) - \ 632baba250SYehuda Sadeh (CONGESTION_ON_THRESH(congestion_kb) >> 2)) 642baba250SYehuda Sadeh 65d801327dSJeff Layton static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, 66d801327dSJeff Layton struct page *page, void **_fsdata); 67d801327dSJeff Layton 6861600ef8SYan, Zheng static inline struct ceph_snap_context *page_snap_context(struct page *page) 6961600ef8SYan, Zheng { 7061600ef8SYan, Zheng if (PagePrivate(page)) 7161600ef8SYan, Zheng return (void *)page->private; 7261600ef8SYan, Zheng return NULL; 7361600ef8SYan, Zheng } 741d3576fdSSage Weil 751d3576fdSSage Weil /* 761d3576fdSSage Weil * Dirty a page. Optimistically adjust accounting, on the assumption 771d3576fdSSage Weil * that we won't race with invalidate. If we do, readjust. 781d3576fdSSage Weil */ 791d3576fdSSage Weil static int ceph_set_page_dirty(struct page *page) 801d3576fdSSage Weil { 811d3576fdSSage Weil struct address_space *mapping = page->mapping; 821d3576fdSSage Weil struct inode *inode; 831d3576fdSSage Weil struct ceph_inode_info *ci; 841d3576fdSSage Weil struct ceph_snap_context *snapc; 857d6e1f54SSha Zhengju int ret; 861d3576fdSSage Weil 871d3576fdSSage Weil if (unlikely(!mapping)) 881d3576fdSSage Weil return !TestSetPageDirty(page); 891d3576fdSSage Weil 907d6e1f54SSha Zhengju if (PageDirty(page)) { 911d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu -- already dirty\n", 921d3576fdSSage Weil mapping->host, page, page->index); 937d6e1f54SSha Zhengju BUG_ON(!PagePrivate(page)); 941d3576fdSSage Weil return 0; 951d3576fdSSage Weil } 961d3576fdSSage Weil 971d3576fdSSage Weil inode = mapping->host; 981d3576fdSSage Weil ci = ceph_inode(inode); 991d3576fdSSage Weil 1001d3576fdSSage Weil /* dirty the head */ 101be655596SSage Weil spin_lock(&ci->i_ceph_lock); 1025dda377cSYan, Zheng BUG_ON(ci->i_wr_ref == 0); // caller should hold Fw reference 1035dda377cSYan, Zheng if (__ceph_have_pending_cap_snap(ci)) { 1045dda377cSYan, Zheng struct ceph_cap_snap *capsnap = 1055dda377cSYan, Zheng list_last_entry(&ci->i_cap_snaps, 1065dda377cSYan, Zheng struct ceph_cap_snap, 1075dda377cSYan, Zheng ci_item); 1085dda377cSYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 1095dda377cSYan, Zheng capsnap->dirty_pages++; 1105dda377cSYan, Zheng } else { 1115dda377cSYan, Zheng BUG_ON(!ci->i_head_snapc); 1125dda377cSYan, Zheng snapc = ceph_get_snap_context(ci->i_head_snapc); 1131d3576fdSSage Weil ++ci->i_wrbuffer_ref_head; 1145dda377cSYan, Zheng } 1151d3576fdSSage Weil if (ci->i_wrbuffer_ref == 0) 1160444d76aSDave Chinner ihold(inode); 1171d3576fdSSage Weil ++ci->i_wrbuffer_ref; 1181d3576fdSSage Weil dout("%p set_page_dirty %p idx %lu head %d/%d -> %d/%d " 1191d3576fdSSage Weil "snapc %p seq %lld (%d snaps)\n", 1201d3576fdSSage Weil mapping->host, page, page->index, 1211d3576fdSSage Weil ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, 1221d3576fdSSage Weil ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, 1231d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 124be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 1251d3576fdSSage Weil 1261d3576fdSSage Weil /* 1271d3576fdSSage Weil * Reference snap context in page->private. Also set 1281d3576fdSSage Weil * PagePrivate so that we get invalidatepage callback. 1291d3576fdSSage Weil */ 1307d6e1f54SSha Zhengju BUG_ON(PagePrivate(page)); 1311d3576fdSSage Weil page->private = (unsigned long)snapc; 1321d3576fdSSage Weil SetPagePrivate(page); 1331d3576fdSSage Weil 1347d6e1f54SSha Zhengju ret = __set_page_dirty_nobuffers(page); 1357d6e1f54SSha Zhengju WARN_ON(!PageLocked(page)); 1367d6e1f54SSha Zhengju WARN_ON(!page->mapping); 1371d3576fdSSage Weil 1387d6e1f54SSha Zhengju return ret; 1391d3576fdSSage Weil } 1401d3576fdSSage Weil 1411d3576fdSSage Weil /* 1421d3576fdSSage Weil * If we are truncating the full page (i.e. offset == 0), adjust the 1431d3576fdSSage Weil * dirty page counters appropriately. Only called if there is private 1441d3576fdSSage Weil * data on the page. 1451d3576fdSSage Weil */ 146d47992f8SLukas Czerner static void ceph_invalidatepage(struct page *page, unsigned int offset, 147d47992f8SLukas Czerner unsigned int length) 1481d3576fdSSage Weil { 1494ce1e9adSAlexander Beregalov struct inode *inode; 1501d3576fdSSage Weil struct ceph_inode_info *ci; 15161600ef8SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 1521d3576fdSSage Weil 1537c46b318SJeff Layton wait_on_page_fscache(page); 1547c46b318SJeff Layton 1554ce1e9adSAlexander Beregalov inode = page->mapping->host; 156b150f5c1SMilosz Tanski ci = ceph_inode(inode); 157b150f5c1SMilosz Tanski 15809cbfeafSKirill A. Shutemov if (offset != 0 || length != PAGE_SIZE) { 159b150f5c1SMilosz Tanski dout("%p invalidatepage %p idx %lu partial dirty page %u~%u\n", 160b150f5c1SMilosz Tanski inode, page, page->index, offset, length); 161b150f5c1SMilosz Tanski return; 162b150f5c1SMilosz Tanski } 1634ce1e9adSAlexander Beregalov 164b072d774SYan, Zheng WARN_ON(!PageLocked(page)); 16599ccbd22SMilosz Tanski if (!PagePrivate(page)) 16699ccbd22SMilosz Tanski return; 16799ccbd22SMilosz Tanski 168569d39fcSLukas Czerner dout("%p invalidatepage %p idx %lu full dirty page\n", 169569d39fcSLukas Czerner inode, page, page->index); 170b150f5c1SMilosz Tanski 1711d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 1721d3576fdSSage Weil ceph_put_snap_context(snapc); 1731d3576fdSSage Weil page->private = 0; 1741d3576fdSSage Weil ClearPagePrivate(page); 1751d3576fdSSage Weil } 1761d3576fdSSage Weil 1777c46b318SJeff Layton static int ceph_releasepage(struct page *page, gfp_t gfp) 1781d3576fdSSage Weil { 179e55f1a18SNeilBrown dout("%p releasepage %p idx %lu (%sdirty)\n", page->mapping->host, 180e55f1a18SNeilBrown page, page->index, PageDirty(page) ? "" : "not "); 18199ccbd22SMilosz Tanski 1827c46b318SJeff Layton if (PageFsCache(page)) { 1837c46b318SJeff Layton if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) 1847c46b318SJeff Layton return 0; 1857c46b318SJeff Layton wait_on_page_fscache(page); 1867c46b318SJeff Layton } 18799ccbd22SMilosz Tanski return !PagePrivate(page); 1881d3576fdSSage Weil } 1891d3576fdSSage Weil 190f0702876SJeff Layton static void ceph_netfs_expand_readahead(struct netfs_read_request *rreq) 191f0702876SJeff Layton { 192f0702876SJeff Layton struct inode *inode = rreq->mapping->host; 193f0702876SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 194f0702876SJeff Layton struct ceph_file_layout *lo = &ci->i_layout; 195f0702876SJeff Layton u32 blockoff; 196f0702876SJeff Layton u64 blockno; 197f0702876SJeff Layton 198f0702876SJeff Layton /* Expand the start downward */ 199f0702876SJeff Layton blockno = div_u64_rem(rreq->start, lo->stripe_unit, &blockoff); 200f0702876SJeff Layton rreq->start = blockno * lo->stripe_unit; 201f0702876SJeff Layton rreq->len += blockoff; 202f0702876SJeff Layton 203f0702876SJeff Layton /* Now, round up the length to the next block */ 204f0702876SJeff Layton rreq->len = roundup(rreq->len, lo->stripe_unit); 205f0702876SJeff Layton } 206f0702876SJeff Layton 207f0702876SJeff Layton static bool ceph_netfs_clamp_length(struct netfs_read_subrequest *subreq) 208f0702876SJeff Layton { 209f0702876SJeff Layton struct inode *inode = subreq->rreq->mapping->host; 210f0702876SJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 211f0702876SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 212f0702876SJeff Layton u64 objno, objoff; 213f0702876SJeff Layton u32 xlen; 214f0702876SJeff Layton 215f0702876SJeff Layton /* Truncate the extent at the end of the current block */ 216f0702876SJeff Layton ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, 217f0702876SJeff Layton &objno, &objoff, &xlen); 218f0702876SJeff Layton subreq->len = min(xlen, fsc->mount_options->rsize); 219f0702876SJeff Layton return true; 220f0702876SJeff Layton } 221f0702876SJeff Layton 222f0702876SJeff Layton static void finish_netfs_read(struct ceph_osd_request *req) 223f0702876SJeff Layton { 224f0702876SJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(req->r_inode); 225f0702876SJeff Layton struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0); 226f0702876SJeff Layton struct netfs_read_subrequest *subreq = req->r_priv; 227f0702876SJeff Layton int num_pages; 228f0702876SJeff Layton int err = req->r_result; 229f0702876SJeff Layton 230f0702876SJeff Layton ceph_update_read_latency(&fsc->mdsc->metric, req->r_start_latency, 231f0702876SJeff Layton req->r_end_latency, err); 232f0702876SJeff Layton 233f0702876SJeff Layton dout("%s: result %d subreq->len=%zu i_size=%lld\n", __func__, req->r_result, 234f0702876SJeff Layton subreq->len, i_size_read(req->r_inode)); 235f0702876SJeff Layton 236f0702876SJeff Layton /* no object means success but no data */ 237f0702876SJeff Layton if (err == -ENOENT) 238f0702876SJeff Layton err = 0; 239f0702876SJeff Layton else if (err == -EBLOCKLISTED) 240f0702876SJeff Layton fsc->blocklisted = true; 241f0702876SJeff Layton 242f0702876SJeff Layton if (err >= 0 && err < subreq->len) 243f0702876SJeff Layton __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); 244f0702876SJeff Layton 245f0702876SJeff Layton netfs_subreq_terminated(subreq, err, true); 246f0702876SJeff Layton 247f0702876SJeff Layton num_pages = calc_pages_for(osd_data->alignment, osd_data->length); 248f0702876SJeff Layton ceph_put_page_vector(osd_data->pages, num_pages, false); 249f0702876SJeff Layton iput(req->r_inode); 250f0702876SJeff Layton } 251f0702876SJeff Layton 252f0702876SJeff Layton static void ceph_netfs_issue_op(struct netfs_read_subrequest *subreq) 253f0702876SJeff Layton { 254f0702876SJeff Layton struct netfs_read_request *rreq = subreq->rreq; 255f0702876SJeff Layton struct inode *inode = rreq->mapping->host; 256f0702876SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 257f0702876SJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 258f0702876SJeff Layton struct ceph_osd_request *req; 259f0702876SJeff Layton struct ceph_vino vino = ceph_vino(inode); 260f0702876SJeff Layton struct iov_iter iter; 261f0702876SJeff Layton struct page **pages; 262f0702876SJeff Layton size_t page_off; 263f0702876SJeff Layton int err = 0; 264f0702876SJeff Layton u64 len = subreq->len; 265f0702876SJeff Layton 266f0702876SJeff Layton req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, subreq->start, &len, 267f0702876SJeff Layton 0, 1, CEPH_OSD_OP_READ, 268f0702876SJeff Layton CEPH_OSD_FLAG_READ | fsc->client->osdc.client->options->read_from_replica, 269f0702876SJeff Layton NULL, ci->i_truncate_seq, ci->i_truncate_size, false); 270f0702876SJeff Layton if (IS_ERR(req)) { 271f0702876SJeff Layton err = PTR_ERR(req); 272f0702876SJeff Layton req = NULL; 273f0702876SJeff Layton goto out; 274f0702876SJeff Layton } 275f0702876SJeff Layton 276f0702876SJeff Layton dout("%s: pos=%llu orig_len=%zu len=%llu\n", __func__, subreq->start, subreq->len, len); 277f0702876SJeff Layton iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages, subreq->start, len); 278f0702876SJeff Layton err = iov_iter_get_pages_alloc(&iter, &pages, len, &page_off); 279f0702876SJeff Layton if (err < 0) { 280f0702876SJeff Layton dout("%s: iov_ter_get_pages_alloc returned %d\n", __func__, err); 281f0702876SJeff Layton goto out; 282f0702876SJeff Layton } 283f0702876SJeff Layton 284f0702876SJeff Layton /* should always give us a page-aligned read */ 285f0702876SJeff Layton WARN_ON_ONCE(page_off); 286f0702876SJeff Layton len = err; 287f0702876SJeff Layton 288f0702876SJeff Layton osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false); 289f0702876SJeff Layton req->r_callback = finish_netfs_read; 290f0702876SJeff Layton req->r_priv = subreq; 291f0702876SJeff Layton req->r_inode = inode; 292f0702876SJeff Layton ihold(inode); 293f0702876SJeff Layton 294f0702876SJeff Layton err = ceph_osdc_start_request(req->r_osdc, req, false); 295f0702876SJeff Layton if (err) 296f0702876SJeff Layton iput(inode); 297f0702876SJeff Layton out: 298f0702876SJeff Layton ceph_osdc_put_request(req); 299f0702876SJeff Layton if (err) 300f0702876SJeff Layton netfs_subreq_terminated(subreq, err, false); 301f0702876SJeff Layton dout("%s: result %d\n", __func__, err); 302f0702876SJeff Layton } 303f0702876SJeff Layton 304f0702876SJeff Layton static void ceph_init_rreq(struct netfs_read_request *rreq, struct file *file) 305f0702876SJeff Layton { 306f0702876SJeff Layton } 307f0702876SJeff Layton 308*49870056SJeff Layton static void ceph_readahead_cleanup(struct address_space *mapping, void *priv) 309*49870056SJeff Layton { 310*49870056SJeff Layton struct inode *inode = mapping->host; 311*49870056SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 312*49870056SJeff Layton int got = (uintptr_t)priv; 313*49870056SJeff Layton 314*49870056SJeff Layton if (got) 315*49870056SJeff Layton ceph_put_cap_refs(ci, got); 316*49870056SJeff Layton } 317*49870056SJeff Layton 318f0702876SJeff Layton const struct netfs_read_request_ops ceph_netfs_read_ops = { 319f0702876SJeff Layton .init_rreq = ceph_init_rreq, 320f0702876SJeff Layton .is_cache_enabled = ceph_is_cache_enabled, 321f0702876SJeff Layton .begin_cache_operation = ceph_begin_cache_operation, 322f0702876SJeff Layton .issue_op = ceph_netfs_issue_op, 323f0702876SJeff Layton .expand_readahead = ceph_netfs_expand_readahead, 324f0702876SJeff Layton .clamp_length = ceph_netfs_clamp_length, 325d801327dSJeff Layton .check_write_begin = ceph_netfs_check_write_begin, 326*49870056SJeff Layton .cleanup = ceph_readahead_cleanup, 327f0702876SJeff Layton }; 328f0702876SJeff Layton 329f0702876SJeff Layton /* read a single page, without unlocking it. */ 330f0702876SJeff Layton static int ceph_readpage(struct file *file, struct page *page) 331f0702876SJeff Layton { 332f0702876SJeff Layton struct inode *inode = file_inode(file); 333f0702876SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 334f0702876SJeff Layton struct ceph_vino vino = ceph_vino(inode); 335f0702876SJeff Layton u64 off = page_offset(page); 336f0702876SJeff Layton u64 len = PAGE_SIZE; 337f0702876SJeff Layton 338f0702876SJeff Layton if (ci->i_inline_version != CEPH_INLINE_NONE) { 339f0702876SJeff Layton /* 340f0702876SJeff Layton * Uptodate inline data should have been added 341f0702876SJeff Layton * into page cache while getting Fcr caps. 342f0702876SJeff Layton */ 343f0702876SJeff Layton if (off == 0) { 344f0702876SJeff Layton unlock_page(page); 345f0702876SJeff Layton return -EINVAL; 346f0702876SJeff Layton } 347f0702876SJeff Layton zero_user_segment(page, 0, PAGE_SIZE); 348f0702876SJeff Layton SetPageUptodate(page); 349f0702876SJeff Layton unlock_page(page); 350f0702876SJeff Layton return 0; 351f0702876SJeff Layton } 352f0702876SJeff Layton 353f0702876SJeff Layton dout("readpage ino %llx.%llx file %p off %llu len %llu page %p index %lu\n", 354f0702876SJeff Layton vino.ino, vino.snap, file, off, len, page, page->index); 355f0702876SJeff Layton 356f0702876SJeff Layton return netfs_readpage(file, page, &ceph_netfs_read_ops, NULL); 357f0702876SJeff Layton } 358f0702876SJeff Layton 359*49870056SJeff Layton static void ceph_readahead(struct readahead_control *ractl) 3601d3576fdSSage Weil { 361*49870056SJeff Layton struct inode *inode = file_inode(ractl->file); 362*49870056SJeff Layton struct ceph_file_info *fi = ractl->file->private_data; 363*49870056SJeff Layton struct ceph_rw_context *rw_ctx; 3642b1ac852SYan, Zheng int got = 0; 3652b1ac852SYan, Zheng int ret = 0; 3662b1ac852SYan, Zheng 36783701246SYan, Zheng if (ceph_inode(inode)->i_inline_version != CEPH_INLINE_NONE) 368*49870056SJeff Layton return; 36983701246SYan, Zheng 37073737682SChengguang Xu rw_ctx = ceph_find_rw_context(fi); 371*49870056SJeff Layton if (!rw_ctx) { 372*49870056SJeff Layton /* 373*49870056SJeff Layton * readahead callers do not necessarily hold Fcb caps 374*49870056SJeff Layton * (e.g. fadvise, madvise). 375*49870056SJeff Layton */ 376*49870056SJeff Layton int want = CEPH_CAP_FILE_CACHE; 377*49870056SJeff Layton 378*49870056SJeff Layton ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got); 379*49870056SJeff Layton if (ret < 0) 380*49870056SJeff Layton dout("start_read %p, error getting cap\n", inode); 381*49870056SJeff Layton else if (!(got & want)) 382*49870056SJeff Layton dout("start_read %p, no cache cap\n", inode); 383*49870056SJeff Layton 384*49870056SJeff Layton if (ret <= 0) 385*49870056SJeff Layton return; 3861d3576fdSSage Weil } 387*49870056SJeff Layton netfs_readahead(ractl, &ceph_netfs_read_ops, (void *)(uintptr_t)got); 3881d3576fdSSage Weil } 3891d3576fdSSage Weil 3901f934b00SYan, Zheng struct ceph_writeback_ctl 3911f934b00SYan, Zheng { 3921f934b00SYan, Zheng loff_t i_size; 3931f934b00SYan, Zheng u64 truncate_size; 3941f934b00SYan, Zheng u32 truncate_seq; 3951f934b00SYan, Zheng bool size_stable; 3962a2d927eSYan, Zheng bool head_snapc; 3971f934b00SYan, Zheng }; 3981f934b00SYan, Zheng 3991d3576fdSSage Weil /* 4001d3576fdSSage Weil * Get ref for the oldest snapc for an inode with dirty data... that is, the 4011d3576fdSSage Weil * only snap context we are allowed to write back. 4021d3576fdSSage Weil */ 4031f934b00SYan, Zheng static struct ceph_snap_context * 40405455e11SYan, Zheng get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, 40505455e11SYan, Zheng struct ceph_snap_context *page_snapc) 4061d3576fdSSage Weil { 4071d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 4081d3576fdSSage Weil struct ceph_snap_context *snapc = NULL; 4091d3576fdSSage Weil struct ceph_cap_snap *capsnap = NULL; 4101d3576fdSSage Weil 411be655596SSage Weil spin_lock(&ci->i_ceph_lock); 4121d3576fdSSage Weil list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 4131d3576fdSSage Weil dout(" cap_snap %p snapc %p has %d dirty pages\n", capsnap, 4141d3576fdSSage Weil capsnap->context, capsnap->dirty_pages); 41505455e11SYan, Zheng if (!capsnap->dirty_pages) 41605455e11SYan, Zheng continue; 41705455e11SYan, Zheng 41805455e11SYan, Zheng /* get i_size, truncate_{seq,size} for page_snapc? */ 41905455e11SYan, Zheng if (snapc && capsnap->context != page_snapc) 42005455e11SYan, Zheng continue; 42105455e11SYan, Zheng 4221f934b00SYan, Zheng if (ctl) { 4231f934b00SYan, Zheng if (capsnap->writing) { 4241f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 4251f934b00SYan, Zheng ctl->size_stable = false; 4261f934b00SYan, Zheng } else { 4271f934b00SYan, Zheng ctl->i_size = capsnap->size; 4281f934b00SYan, Zheng ctl->size_stable = true; 4291f934b00SYan, Zheng } 4301f934b00SYan, Zheng ctl->truncate_size = capsnap->truncate_size; 4311f934b00SYan, Zheng ctl->truncate_seq = capsnap->truncate_seq; 4322a2d927eSYan, Zheng ctl->head_snapc = false; 4331f934b00SYan, Zheng } 43405455e11SYan, Zheng 43505455e11SYan, Zheng if (snapc) 4361d3576fdSSage Weil break; 43705455e11SYan, Zheng 43805455e11SYan, Zheng snapc = ceph_get_snap_context(capsnap->context); 43905455e11SYan, Zheng if (!page_snapc || 44005455e11SYan, Zheng page_snapc == snapc || 44105455e11SYan, Zheng page_snapc->seq > snapc->seq) 44205455e11SYan, Zheng break; 4431d3576fdSSage Weil } 4447d8cb26dSSage Weil if (!snapc && ci->i_wrbuffer_ref_head) { 44580e755feSSage Weil snapc = ceph_get_snap_context(ci->i_head_snapc); 4461d3576fdSSage Weil dout(" head snapc %p has %d dirty pages\n", 4471d3576fdSSage Weil snapc, ci->i_wrbuffer_ref_head); 4481f934b00SYan, Zheng if (ctl) { 4491f934b00SYan, Zheng ctl->i_size = i_size_read(inode); 4501f934b00SYan, Zheng ctl->truncate_size = ci->i_truncate_size; 4511f934b00SYan, Zheng ctl->truncate_seq = ci->i_truncate_seq; 4521f934b00SYan, Zheng ctl->size_stable = false; 4532a2d927eSYan, Zheng ctl->head_snapc = true; 4541f934b00SYan, Zheng } 4551d3576fdSSage Weil } 456be655596SSage Weil spin_unlock(&ci->i_ceph_lock); 4571d3576fdSSage Weil return snapc; 4581d3576fdSSage Weil } 4591d3576fdSSage Weil 4601f934b00SYan, Zheng static u64 get_writepages_data_length(struct inode *inode, 4611f934b00SYan, Zheng struct page *page, u64 start) 4621f934b00SYan, Zheng { 4631f934b00SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 4641f934b00SYan, Zheng struct ceph_snap_context *snapc = page_snap_context(page); 4651f934b00SYan, Zheng struct ceph_cap_snap *capsnap = NULL; 4661f934b00SYan, Zheng u64 end = i_size_read(inode); 4671f934b00SYan, Zheng 4681f934b00SYan, Zheng if (snapc != ci->i_head_snapc) { 4691f934b00SYan, Zheng bool found = false; 4701f934b00SYan, Zheng spin_lock(&ci->i_ceph_lock); 4711f934b00SYan, Zheng list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { 4721f934b00SYan, Zheng if (capsnap->context == snapc) { 4731f934b00SYan, Zheng if (!capsnap->writing) 4741f934b00SYan, Zheng end = capsnap->size; 4751f934b00SYan, Zheng found = true; 4761f934b00SYan, Zheng break; 4771f934b00SYan, Zheng } 4781f934b00SYan, Zheng } 4791f934b00SYan, Zheng spin_unlock(&ci->i_ceph_lock); 4801f934b00SYan, Zheng WARN_ON(!found); 4811f934b00SYan, Zheng } 4821f934b00SYan, Zheng if (end > page_offset(page) + PAGE_SIZE) 4831f934b00SYan, Zheng end = page_offset(page) + PAGE_SIZE; 4841f934b00SYan, Zheng return end > start ? end - start : 0; 4851f934b00SYan, Zheng } 4861f934b00SYan, Zheng 4871d3576fdSSage Weil /* 4881d3576fdSSage Weil * Write a single page, but leave the page locked. 4891d3576fdSSage Weil * 490b72b13ebSJeff Layton * If we get a write error, mark the mapping for error, but still adjust the 4911d3576fdSSage Weil * dirty page accounting (i.e., page is no longer dirty). 4921d3576fdSSage Weil */ 4931d3576fdSSage Weil static int writepage_nounlock(struct page *page, struct writeback_control *wbc) 4941d3576fdSSage Weil { 4956390987fSJeff Layton struct inode *inode = page->mapping->host; 4966390987fSJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 4976390987fSJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 4986298a337SSage Weil struct ceph_snap_context *snapc, *oldest; 499fc2744aaSYan, Zheng loff_t page_off = page_offset(page); 5006390987fSJeff Layton int err; 5016390987fSJeff Layton loff_t len = PAGE_SIZE; 5021f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 5036390987fSJeff Layton struct ceph_osd_client *osdc = &fsc->client->osdc; 5046390987fSJeff Layton struct ceph_osd_request *req; 5051d3576fdSSage Weil 5061d3576fdSSage Weil dout("writepage %p idx %lu\n", page, page->index); 5071d3576fdSSage Weil 5081d3576fdSSage Weil /* verify this is a writeable snap context */ 50961600ef8SYan, Zheng snapc = page_snap_context(page); 510d37b1d99SMarkus Elfring if (!snapc) { 5111d3576fdSSage Weil dout("writepage %p page %p not dirty?\n", inode, page); 51243986881SYan, Zheng return 0; 5131d3576fdSSage Weil } 51405455e11SYan, Zheng oldest = get_oldest_context(inode, &ceph_wbc, snapc); 5156298a337SSage Weil if (snapc->seq > oldest->seq) { 5161d3576fdSSage Weil dout("writepage %p page %p snapc %p not writeable - noop\n", 51761600ef8SYan, Zheng inode, page, snapc); 5181d3576fdSSage Weil /* we should only noop if called by kswapd */ 519fa71fefbSYan, Zheng WARN_ON(!(current->flags & PF_MEMALLOC)); 5206298a337SSage Weil ceph_put_snap_context(oldest); 521fa71fefbSYan, Zheng redirty_page_for_writepage(wbc, page); 52243986881SYan, Zheng return 0; 5231d3576fdSSage Weil } 5246298a337SSage Weil ceph_put_snap_context(oldest); 5251d3576fdSSage Weil 5261d3576fdSSage Weil /* is this a partial page at end of file? */ 5271f934b00SYan, Zheng if (page_off >= ceph_wbc.i_size) { 5281f934b00SYan, Zheng dout("%p page eof %llu\n", page, ceph_wbc.i_size); 52905455e11SYan, Zheng page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE); 53043986881SYan, Zheng return 0; 531fc2744aaSYan, Zheng } 53243986881SYan, Zheng 5331f934b00SYan, Zheng if (ceph_wbc.i_size < page_off + len) 5341f934b00SYan, Zheng len = ceph_wbc.i_size - page_off; 5351d3576fdSSage Weil 5366390987fSJeff Layton dout("writepage %p page %p index %lu on %llu~%llu snapc %p seq %lld\n", 5371c0a9c2dSYan, Zheng inode, page, page->index, page_off, len, snapc, snapc->seq); 5381d3576fdSSage Weil 539314c4737SYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 5403d14c5d2SYehuda Sadeh CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) 54109dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 5422baba250SYehuda Sadeh 5431d3576fdSSage Weil set_page_writeback(page); 5446390987fSJeff Layton req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode), page_off, &len, 0, 1, 5456390987fSJeff Layton CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, snapc, 5466390987fSJeff Layton ceph_wbc.truncate_seq, ceph_wbc.truncate_size, 5476390987fSJeff Layton true); 5486390987fSJeff Layton if (IS_ERR(req)) { 5496390987fSJeff Layton redirty_page_for_writepage(wbc, page); 5506390987fSJeff Layton end_page_writeback(page); 5516390987fSJeff Layton return PTR_ERR(req); 5526390987fSJeff Layton } 5536390987fSJeff Layton 5546390987fSJeff Layton /* it may be a short write due to an object boundary */ 5556390987fSJeff Layton WARN_ON_ONCE(len > PAGE_SIZE); 5566390987fSJeff Layton osd_req_op_extent_osd_data_pages(req, 0, &page, len, 0, false, false); 5576390987fSJeff Layton dout("writepage %llu~%llu (%llu bytes)\n", page_off, len, len); 5586390987fSJeff Layton 5596390987fSJeff Layton req->r_mtime = inode->i_mtime; 5606390987fSJeff Layton err = ceph_osdc_start_request(osdc, req, true); 5616390987fSJeff Layton if (!err) 5626390987fSJeff Layton err = ceph_osdc_wait_request(osdc, req); 5636390987fSJeff Layton 5646390987fSJeff Layton ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 5656390987fSJeff Layton req->r_end_latency, err); 5666390987fSJeff Layton 5676390987fSJeff Layton ceph_osdc_put_request(req); 5686390987fSJeff Layton if (err == 0) 5696390987fSJeff Layton err = len; 5706390987fSJeff Layton 5711d3576fdSSage Weil if (err < 0) { 572ad15ec06SYan, Zheng struct writeback_control tmp_wbc; 573ad15ec06SYan, Zheng if (!wbc) 574ad15ec06SYan, Zheng wbc = &tmp_wbc; 575ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 576ad15ec06SYan, Zheng /* killed by SIGKILL */ 577ad15ec06SYan, Zheng dout("writepage interrupted page %p\n", page); 578ad15ec06SYan, Zheng redirty_page_for_writepage(wbc, page); 579ad15ec06SYan, Zheng end_page_writeback(page); 58043986881SYan, Zheng return err; 581ad15ec06SYan, Zheng } 5820b98acd6SIlya Dryomov if (err == -EBLOCKLISTED) 5830b98acd6SIlya Dryomov fsc->blocklisted = true; 584ad15ec06SYan, Zheng dout("writepage setting page/mapping error %d %p\n", 585ad15ec06SYan, Zheng err, page); 5861d3576fdSSage Weil mapping_set_error(&inode->i_data, err); 5871d3576fdSSage Weil wbc->pages_skipped++; 5881d3576fdSSage Weil } else { 5891d3576fdSSage Weil dout("writepage cleaned page %p\n", page); 5901d3576fdSSage Weil err = 0; /* vfs expects us to return 0 */ 5911d3576fdSSage Weil } 5921d3576fdSSage Weil page->private = 0; 5931d3576fdSSage Weil ClearPagePrivate(page); 5941d3576fdSSage Weil end_page_writeback(page); 5951d3576fdSSage Weil ceph_put_wrbuffer_cap_refs(ci, 1, snapc); 5966298a337SSage Weil ceph_put_snap_context(snapc); /* page's reference */ 597314c4737SYan, Zheng 598314c4737SYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 599314c4737SYan, Zheng CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) 600314c4737SYan, Zheng clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 601314c4737SYan, Zheng 6021d3576fdSSage Weil return err; 6031d3576fdSSage Weil } 6041d3576fdSSage Weil 6051d3576fdSSage Weil static int ceph_writepage(struct page *page, struct writeback_control *wbc) 6061d3576fdSSage Weil { 607dbd646a8SYehuda Sadeh int err; 608dbd646a8SYehuda Sadeh struct inode *inode = page->mapping->host; 609dbd646a8SYehuda Sadeh BUG_ON(!inode); 61070b666c3SSage Weil ihold(inode); 611dbd646a8SYehuda Sadeh err = writepage_nounlock(page, wbc); 612ad15ec06SYan, Zheng if (err == -ERESTARTSYS) { 613ad15ec06SYan, Zheng /* direct memory reclaimer was killed by SIGKILL. return 0 614ad15ec06SYan, Zheng * to prevent caller from setting mapping/page error */ 615ad15ec06SYan, Zheng err = 0; 616ad15ec06SYan, Zheng } 6171d3576fdSSage Weil unlock_page(page); 618dbd646a8SYehuda Sadeh iput(inode); 6191d3576fdSSage Weil return err; 6201d3576fdSSage Weil } 6211d3576fdSSage Weil 6221d3576fdSSage Weil /* 6231d3576fdSSage Weil * async writeback completion handler. 6241d3576fdSSage Weil * 6251d3576fdSSage Weil * If we get an error, set the mapping error bit, but not the individual 6261d3576fdSSage Weil * page error bits. 6271d3576fdSSage Weil */ 62885e084feSIlya Dryomov static void writepages_finish(struct ceph_osd_request *req) 6291d3576fdSSage Weil { 6301d3576fdSSage Weil struct inode *inode = req->r_inode; 6311d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 63287060c10SAlex Elder struct ceph_osd_data *osd_data; 6331d3576fdSSage Weil struct page *page; 6345b64640cSYan, Zheng int num_pages, total_pages = 0; 6355b64640cSYan, Zheng int i, j; 6365b64640cSYan, Zheng int rc = req->r_result; 6371d3576fdSSage Weil struct ceph_snap_context *snapc = req->r_snapc; 6381d3576fdSSage Weil struct address_space *mapping = inode->i_mapping; 6393d14c5d2SYehuda Sadeh struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 6405b64640cSYan, Zheng bool remove_page; 6411d3576fdSSage Weil 6425b64640cSYan, Zheng dout("writepages_finish %p rc %d\n", inode, rc); 64326544c62SJeff Layton if (rc < 0) { 6441d3576fdSSage Weil mapping_set_error(mapping, rc); 64526544c62SJeff Layton ceph_set_error_write(ci); 6460b98acd6SIlya Dryomov if (rc == -EBLOCKLISTED) 6470b98acd6SIlya Dryomov fsc->blocklisted = true; 64826544c62SJeff Layton } else { 64926544c62SJeff Layton ceph_clear_error_write(ci); 65026544c62SJeff Layton } 651e63dc5c7SYehuda Sadeh 65297e27aaaSXiubo Li ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 65397e27aaaSXiubo Li req->r_end_latency, rc); 65497e27aaaSXiubo Li 655e63dc5c7SYehuda Sadeh /* 656e63dc5c7SYehuda Sadeh * We lost the cache cap, need to truncate the page before 657e63dc5c7SYehuda Sadeh * it is unlocked, otherwise we'd truncate it later in the 658e63dc5c7SYehuda Sadeh * page truncation thread, possibly losing some data that 659e63dc5c7SYehuda Sadeh * raced its way in 660e63dc5c7SYehuda Sadeh */ 6615b64640cSYan, Zheng remove_page = !(ceph_caps_issued(ci) & 6625b64640cSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); 6635b64640cSYan, Zheng 6645b64640cSYan, Zheng /* clean all pages */ 6655b64640cSYan, Zheng for (i = 0; i < req->r_num_ops; i++) { 6665b64640cSYan, Zheng if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) 6675b64640cSYan, Zheng break; 6685b64640cSYan, Zheng 6695b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, i); 6705b64640cSYan, Zheng BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); 6715b64640cSYan, Zheng num_pages = calc_pages_for((u64)osd_data->alignment, 6725b64640cSYan, Zheng (u64)osd_data->length); 6735b64640cSYan, Zheng total_pages += num_pages; 6745b64640cSYan, Zheng for (j = 0; j < num_pages; j++) { 6755b64640cSYan, Zheng page = osd_data->pages[j]; 6765b64640cSYan, Zheng BUG_ON(!page); 6775b64640cSYan, Zheng WARN_ON(!PageUptodate(page)); 6785b64640cSYan, Zheng 6795b64640cSYan, Zheng if (atomic_long_dec_return(&fsc->writeback_count) < 6805b64640cSYan, Zheng CONGESTION_OFF_THRESH( 6815b64640cSYan, Zheng fsc->mount_options->congestion_kb)) 68209dc9fc2SJan Kara clear_bdi_congested(inode_to_bdi(inode), 6835b64640cSYan, Zheng BLK_RW_ASYNC); 6845b64640cSYan, Zheng 6855b64640cSYan, Zheng ceph_put_snap_context(page_snap_context(page)); 6865b64640cSYan, Zheng page->private = 0; 6875b64640cSYan, Zheng ClearPagePrivate(page); 6885b64640cSYan, Zheng dout("unlocking %p\n", page); 6895b64640cSYan, Zheng end_page_writeback(page); 6905b64640cSYan, Zheng 6915b64640cSYan, Zheng if (remove_page) 6925b64640cSYan, Zheng generic_error_remove_page(inode->i_mapping, 6935b64640cSYan, Zheng page); 694e63dc5c7SYehuda Sadeh 6951d3576fdSSage Weil unlock_page(page); 6961d3576fdSSage Weil } 6975b64640cSYan, Zheng dout("writepages_finish %p wrote %llu bytes cleaned %d pages\n", 6985b64640cSYan, Zheng inode, osd_data->length, rc >= 0 ? num_pages : 0); 6991d3576fdSSage Weil 70096ac9158SJohn Hubbard release_pages(osd_data->pages, num_pages); 7015b64640cSYan, Zheng } 7025b64640cSYan, Zheng 7035b64640cSYan, Zheng ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); 7045b64640cSYan, Zheng 7055b64640cSYan, Zheng osd_data = osd_req_op_extent_osd_data(req, 0); 70687060c10SAlex Elder if (osd_data->pages_from_pool) 707a0102bdaSJeff Layton mempool_free(osd_data->pages, ceph_wb_pagevec_pool); 7081d3576fdSSage Weil else 70987060c10SAlex Elder kfree(osd_data->pages); 7101d3576fdSSage Weil ceph_osdc_put_request(req); 7111d3576fdSSage Weil } 7121d3576fdSSage Weil 7131d3576fdSSage Weil /* 7141d3576fdSSage Weil * initiate async writeback 7151d3576fdSSage Weil */ 7161d3576fdSSage Weil static int ceph_writepages_start(struct address_space *mapping, 7171d3576fdSSage Weil struct writeback_control *wbc) 7181d3576fdSSage Weil { 7191d3576fdSSage Weil struct inode *inode = mapping->host; 7201d3576fdSSage Weil struct ceph_inode_info *ci = ceph_inode(inode); 721fc2744aaSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 722fc2744aaSYan, Zheng struct ceph_vino vino = ceph_vino(inode); 7232a2d927eSYan, Zheng pgoff_t index, start_index, end = -1; 72480e755feSSage Weil struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; 7251d3576fdSSage Weil struct pagevec pvec; 7261d3576fdSSage Weil int rc = 0; 72793407472SFabian Frederick unsigned int wsize = i_blocksize(inode); 7281d3576fdSSage Weil struct ceph_osd_request *req = NULL; 7291f934b00SYan, Zheng struct ceph_writeback_ctl ceph_wbc; 730590e9d98SYan, Zheng bool should_loop, range_whole = false; 731af9cc401SYan, Zheng bool done = false; 7321d3576fdSSage Weil 7333fb99d48SYanhu Cao dout("writepages_start %p (mode=%s)\n", inode, 7341d3576fdSSage Weil wbc->sync_mode == WB_SYNC_NONE ? "NONE" : 7351d3576fdSSage Weil (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); 7361d3576fdSSage Weil 73750c9132dSJeff Layton if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) { 7386c93df5dSYan, Zheng if (ci->i_wrbuffer_ref > 0) { 7396c93df5dSYan, Zheng pr_warn_ratelimited( 7406c93df5dSYan, Zheng "writepage_start %p %lld forced umount\n", 7416c93df5dSYan, Zheng inode, ceph_ino(inode)); 7426c93df5dSYan, Zheng } 743a341d4dfSYan, Zheng mapping_set_error(mapping, -EIO); 7441d3576fdSSage Weil return -EIO; /* we're in a forced umount, don't write! */ 7451d3576fdSSage Weil } 74695cca2b4SYan, Zheng if (fsc->mount_options->wsize < wsize) 7473d14c5d2SYehuda Sadeh wsize = fsc->mount_options->wsize; 7481d3576fdSSage Weil 74986679820SMel Gorman pagevec_init(&pvec); 7501d3576fdSSage Weil 751590e9d98SYan, Zheng start_index = wbc->range_cyclic ? mapping->writeback_index : 0; 752590e9d98SYan, Zheng index = start_index; 7531d3576fdSSage Weil 7541d3576fdSSage Weil retry: 7551d3576fdSSage Weil /* find oldest snap context with dirty data */ 75605455e11SYan, Zheng snapc = get_oldest_context(inode, &ceph_wbc, NULL); 7571d3576fdSSage Weil if (!snapc) { 7581d3576fdSSage Weil /* hmm, why does writepages get called when there 7591d3576fdSSage Weil is no dirty data? */ 7601d3576fdSSage Weil dout(" no snap context with dirty data?\n"); 7611d3576fdSSage Weil goto out; 7621d3576fdSSage Weil } 7631d3576fdSSage Weil dout(" oldest snapc is %p seq %lld (%d snaps)\n", 7641d3576fdSSage Weil snapc, snapc->seq, snapc->num_snaps); 765fc2744aaSYan, Zheng 7662a2d927eSYan, Zheng should_loop = false; 7672a2d927eSYan, Zheng if (ceph_wbc.head_snapc && snapc != last_snapc) { 7682a2d927eSYan, Zheng /* where to start/end? */ 7692a2d927eSYan, Zheng if (wbc->range_cyclic) { 7702a2d927eSYan, Zheng index = start_index; 7712a2d927eSYan, Zheng end = -1; 7722a2d927eSYan, Zheng if (index > 0) 7732a2d927eSYan, Zheng should_loop = true; 7742a2d927eSYan, Zheng dout(" cyclic, start at %lu\n", index); 7752a2d927eSYan, Zheng } else { 7762a2d927eSYan, Zheng index = wbc->range_start >> PAGE_SHIFT; 7772a2d927eSYan, Zheng end = wbc->range_end >> PAGE_SHIFT; 7782a2d927eSYan, Zheng if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 7792a2d927eSYan, Zheng range_whole = true; 7802a2d927eSYan, Zheng dout(" not cyclic, %lu to %lu\n", index, end); 7811d3576fdSSage Weil } 7822a2d927eSYan, Zheng } else if (!ceph_wbc.head_snapc) { 7832a2d927eSYan, Zheng /* Do not respect wbc->range_{start,end}. Dirty pages 7842a2d927eSYan, Zheng * in that range can be associated with newer snapc. 7852a2d927eSYan, Zheng * They are not writeable until we write all dirty pages 7862a2d927eSYan, Zheng * associated with 'snapc' get written */ 7871582af2eSYan, Zheng if (index > 0) 7882a2d927eSYan, Zheng should_loop = true; 7892a2d927eSYan, Zheng dout(" non-head snapc, range whole\n"); 7902a2d927eSYan, Zheng } 7912a2d927eSYan, Zheng 7922a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 7931d3576fdSSage Weil last_snapc = snapc; 7941d3576fdSSage Weil 795af9cc401SYan, Zheng while (!done && index <= end) { 7965b64640cSYan, Zheng int num_ops = 0, op_idx; 7970e5ecac7SYan, Zheng unsigned i, pvec_pages, max_pages, locked_pages = 0; 7985b64640cSYan, Zheng struct page **pages = NULL, **data_pages; 7991d3576fdSSage Weil struct page *page; 8000e5ecac7SYan, Zheng pgoff_t strip_unit_end = 0; 8015b64640cSYan, Zheng u64 offset = 0, len = 0; 802a0102bdaSJeff Layton bool from_pool = false; 8031d3576fdSSage Weil 8040e5ecac7SYan, Zheng max_pages = wsize >> PAGE_SHIFT; 8051d3576fdSSage Weil 8061d3576fdSSage Weil get_more_pages: 8072e169296SJeff Layton pvec_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, 8082e169296SJeff Layton end, PAGECACHE_TAG_DIRTY); 8090ed75fc8SJan Kara dout("pagevec_lookup_range_tag got %d\n", pvec_pages); 8101d3576fdSSage Weil if (!pvec_pages && !locked_pages) 8111d3576fdSSage Weil break; 8121d3576fdSSage Weil for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { 8131d3576fdSSage Weil page = pvec.pages[i]; 8141d3576fdSSage Weil dout("? %p idx %lu\n", page, page->index); 8151d3576fdSSage Weil if (locked_pages == 0) 8161d3576fdSSage Weil lock_page(page); /* first page */ 8171d3576fdSSage Weil else if (!trylock_page(page)) 8181d3576fdSSage Weil break; 8191d3576fdSSage Weil 8201d3576fdSSage Weil /* only dirty pages, or our accounting breaks */ 8211d3576fdSSage Weil if (unlikely(!PageDirty(page)) || 8221d3576fdSSage Weil unlikely(page->mapping != mapping)) { 8231d3576fdSSage Weil dout("!dirty or !mapping %p\n", page); 8241d3576fdSSage Weil unlock_page(page); 8250713e5f2SYan, Zheng continue; 8261d3576fdSSage Weil } 827af9cc401SYan, Zheng /* only if matching snap context */ 828af9cc401SYan, Zheng pgsnapc = page_snap_context(page); 829af9cc401SYan, Zheng if (pgsnapc != snapc) { 830af9cc401SYan, Zheng dout("page snapc %p %lld != oldest %p %lld\n", 831af9cc401SYan, Zheng pgsnapc, pgsnapc->seq, snapc, snapc->seq); 8321582af2eSYan, Zheng if (!should_loop && 8331582af2eSYan, Zheng !ceph_wbc.head_snapc && 8341582af2eSYan, Zheng wbc->sync_mode != WB_SYNC_NONE) 8351582af2eSYan, Zheng should_loop = true; 8361d3576fdSSage Weil unlock_page(page); 837af9cc401SYan, Zheng continue; 8381d3576fdSSage Weil } 8391f934b00SYan, Zheng if (page_offset(page) >= ceph_wbc.i_size) { 8401f934b00SYan, Zheng dout("%p page eof %llu\n", 8411f934b00SYan, Zheng page, ceph_wbc.i_size); 842c95f1c5fSErqi Chen if ((ceph_wbc.size_stable || 843c95f1c5fSErqi Chen page_offset(page) >= i_size_read(inode)) && 844c95f1c5fSErqi Chen clear_page_dirty_for_io(page)) 845af9cc401SYan, Zheng mapping->a_ops->invalidatepage(page, 846af9cc401SYan, Zheng 0, PAGE_SIZE); 847af9cc401SYan, Zheng unlock_page(page); 848af9cc401SYan, Zheng continue; 849af9cc401SYan, Zheng } 850af9cc401SYan, Zheng if (strip_unit_end && (page->index > strip_unit_end)) { 851af9cc401SYan, Zheng dout("end of strip unit %p\n", page); 8521d3576fdSSage Weil unlock_page(page); 8531d3576fdSSage Weil break; 8541d3576fdSSage Weil } 8551d3576fdSSage Weil if (PageWriteback(page)) { 8560713e5f2SYan, Zheng if (wbc->sync_mode == WB_SYNC_NONE) { 8571d3576fdSSage Weil dout("%p under writeback\n", page); 8581d3576fdSSage Weil unlock_page(page); 8590713e5f2SYan, Zheng continue; 8600713e5f2SYan, Zheng } 8610713e5f2SYan, Zheng dout("waiting on writeback %p\n", page); 8620713e5f2SYan, Zheng wait_on_page_writeback(page); 8631d3576fdSSage Weil } 8641d3576fdSSage Weil 8651d3576fdSSage Weil if (!clear_page_dirty_for_io(page)) { 8661d3576fdSSage Weil dout("%p !clear_page_dirty_for_io\n", page); 8671d3576fdSSage Weil unlock_page(page); 8680713e5f2SYan, Zheng continue; 8691d3576fdSSage Weil } 8701d3576fdSSage Weil 871e5975c7cSAlex Elder /* 872e5975c7cSAlex Elder * We have something to write. If this is 873e5975c7cSAlex Elder * the first locked page this time through, 8745b64640cSYan, Zheng * calculate max possinle write size and 8755b64640cSYan, Zheng * allocate a page array 876e5975c7cSAlex Elder */ 8771d3576fdSSage Weil if (locked_pages == 0) { 8785b64640cSYan, Zheng u64 objnum; 8795b64640cSYan, Zheng u64 objoff; 880dccbf080SIlya Dryomov u32 xlen; 8815b64640cSYan, Zheng 8821d3576fdSSage Weil /* prepare async write request */ 8836285bc23SAlex Elder offset = (u64)page_offset(page); 884dccbf080SIlya Dryomov ceph_calc_file_object_mapping(&ci->i_layout, 885dccbf080SIlya Dryomov offset, wsize, 8865b64640cSYan, Zheng &objnum, &objoff, 887dccbf080SIlya Dryomov &xlen); 888dccbf080SIlya Dryomov len = xlen; 8898c71897bSHenry C Chang 8903fb99d48SYanhu Cao num_ops = 1; 8915b64640cSYan, Zheng strip_unit_end = page->index + 89209cbfeafSKirill A. Shutemov ((len - 1) >> PAGE_SHIFT); 893715e4cd4SYan, Zheng 8945b64640cSYan, Zheng BUG_ON(pages); 89588486957SAlex Elder max_pages = calc_pages_for(0, (u64)len); 8966da2ec56SKees Cook pages = kmalloc_array(max_pages, 8976da2ec56SKees Cook sizeof(*pages), 898fc2744aaSYan, Zheng GFP_NOFS); 89988486957SAlex Elder if (!pages) { 900a0102bdaSJeff Layton from_pool = true; 901a0102bdaSJeff Layton pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); 902e5975c7cSAlex Elder BUG_ON(!pages); 90388486957SAlex Elder } 9045b64640cSYan, Zheng 9055b64640cSYan, Zheng len = 0; 9065b64640cSYan, Zheng } else if (page->index != 90709cbfeafSKirill A. Shutemov (offset + len) >> PAGE_SHIFT) { 908a0102bdaSJeff Layton if (num_ops >= (from_pool ? CEPH_OSD_SLAB_OPS : 9095b64640cSYan, Zheng CEPH_OSD_MAX_OPS)) { 9105b64640cSYan, Zheng redirty_page_for_writepage(wbc, page); 9115b64640cSYan, Zheng unlock_page(page); 9125b64640cSYan, Zheng break; 9135b64640cSYan, Zheng } 9145b64640cSYan, Zheng 9155b64640cSYan, Zheng num_ops++; 9165b64640cSYan, Zheng offset = (u64)page_offset(page); 9175b64640cSYan, Zheng len = 0; 9181d3576fdSSage Weil } 9191d3576fdSSage Weil 9201d3576fdSSage Weil /* note position of first page in pvec */ 9211d3576fdSSage Weil dout("%p will write page %p idx %lu\n", 9221d3576fdSSage Weil inode, page, page->index); 9232baba250SYehuda Sadeh 9245b64640cSYan, Zheng if (atomic_long_inc_return(&fsc->writeback_count) > 9255b64640cSYan, Zheng CONGESTION_ON_THRESH( 9263d14c5d2SYehuda Sadeh fsc->mount_options->congestion_kb)) { 92709dc9fc2SJan Kara set_bdi_congested(inode_to_bdi(inode), 928213c99eeSSage Weil BLK_RW_ASYNC); 9292baba250SYehuda Sadeh } 9302baba250SYehuda Sadeh 9310713e5f2SYan, Zheng 9320713e5f2SYan, Zheng pages[locked_pages++] = page; 9330713e5f2SYan, Zheng pvec.pages[i] = NULL; 9340713e5f2SYan, Zheng 93509cbfeafSKirill A. Shutemov len += PAGE_SIZE; 9361d3576fdSSage Weil } 9371d3576fdSSage Weil 9381d3576fdSSage Weil /* did we get anything? */ 9391d3576fdSSage Weil if (!locked_pages) 9401d3576fdSSage Weil goto release_pvec_pages; 9411d3576fdSSage Weil if (i) { 9420713e5f2SYan, Zheng unsigned j, n = 0; 9430713e5f2SYan, Zheng /* shift unused page to beginning of pvec */ 9440713e5f2SYan, Zheng for (j = 0; j < pvec_pages; j++) { 9450713e5f2SYan, Zheng if (!pvec.pages[j]) 9460713e5f2SYan, Zheng continue; 9470713e5f2SYan, Zheng if (n < j) 9480713e5f2SYan, Zheng pvec.pages[n] = pvec.pages[j]; 9490713e5f2SYan, Zheng n++; 9500713e5f2SYan, Zheng } 9510713e5f2SYan, Zheng pvec.nr = n; 9521d3576fdSSage Weil 9531d3576fdSSage Weil if (pvec_pages && i == pvec_pages && 9541d3576fdSSage Weil locked_pages < max_pages) { 9551d3576fdSSage Weil dout("reached end pvec, trying for more\n"); 9560713e5f2SYan, Zheng pagevec_release(&pvec); 9571d3576fdSSage Weil goto get_more_pages; 9581d3576fdSSage Weil } 9591d3576fdSSage Weil } 9601d3576fdSSage Weil 9615b64640cSYan, Zheng new_request: 962e5975c7cSAlex Elder offset = page_offset(pages[0]); 9635b64640cSYan, Zheng len = wsize; 9645b64640cSYan, Zheng 9655b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 9665b64640cSYan, Zheng &ci->i_layout, vino, 9675b64640cSYan, Zheng offset, &len, 0, num_ops, 9681f934b00SYan, Zheng CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 9691f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 9701f934b00SYan, Zheng ceph_wbc.truncate_size, false); 9715b64640cSYan, Zheng if (IS_ERR(req)) { 9725b64640cSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, 9735b64640cSYan, Zheng &ci->i_layout, vino, 9745b64640cSYan, Zheng offset, &len, 0, 9755b64640cSYan, Zheng min(num_ops, 9765b64640cSYan, Zheng CEPH_OSD_SLAB_OPS), 9775b64640cSYan, Zheng CEPH_OSD_OP_WRITE, 97854ea0046SIlya Dryomov CEPH_OSD_FLAG_WRITE, 9791f934b00SYan, Zheng snapc, ceph_wbc.truncate_seq, 9801f934b00SYan, Zheng ceph_wbc.truncate_size, true); 9815b64640cSYan, Zheng BUG_ON(IS_ERR(req)); 9825b64640cSYan, Zheng } 9835b64640cSYan, Zheng BUG_ON(len < page_offset(pages[locked_pages - 1]) + 98409cbfeafSKirill A. Shutemov PAGE_SIZE - offset); 9855b64640cSYan, Zheng 9865b64640cSYan, Zheng req->r_callback = writepages_finish; 9875b64640cSYan, Zheng req->r_inode = inode; 9885b64640cSYan, Zheng 9895b64640cSYan, Zheng /* Format the osd request message and submit the write */ 9905b64640cSYan, Zheng len = 0; 9915b64640cSYan, Zheng data_pages = pages; 9925b64640cSYan, Zheng op_idx = 0; 9935b64640cSYan, Zheng for (i = 0; i < locked_pages; i++) { 9945b64640cSYan, Zheng u64 cur_offset = page_offset(pages[i]); 9955b64640cSYan, Zheng if (offset + len != cur_offset) { 9963fb99d48SYanhu Cao if (op_idx + 1 == req->r_num_ops) 9975b64640cSYan, Zheng break; 9985b64640cSYan, Zheng osd_req_op_extent_dup_last(req, op_idx, 9995b64640cSYan, Zheng cur_offset - offset); 10005b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", 10015b64640cSYan, Zheng offset, len); 10025b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, 10035b64640cSYan, Zheng data_pages, len, 0, 1004a0102bdaSJeff Layton from_pool, false); 10055b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 10065b64640cSYan, Zheng 10075b64640cSYan, Zheng len = 0; 10085b64640cSYan, Zheng offset = cur_offset; 10095b64640cSYan, Zheng data_pages = pages + i; 10105b64640cSYan, Zheng op_idx++; 10115b64640cSYan, Zheng } 10125b64640cSYan, Zheng 10135b64640cSYan, Zheng set_page_writeback(pages[i]); 101409cbfeafSKirill A. Shutemov len += PAGE_SIZE; 10155b64640cSYan, Zheng } 10165b64640cSYan, Zheng 10171f934b00SYan, Zheng if (ceph_wbc.size_stable) { 10181f934b00SYan, Zheng len = min(len, ceph_wbc.i_size - offset); 10195b64640cSYan, Zheng } else if (i == locked_pages) { 1020e1966b49SYan, Zheng /* writepages_finish() clears writeback pages 1021e1966b49SYan, Zheng * according to the data length, so make sure 1022e1966b49SYan, Zheng * data length covers all locked pages */ 102309cbfeafSKirill A. Shutemov u64 min_len = len + 1 - PAGE_SIZE; 10241f934b00SYan, Zheng len = get_writepages_data_length(inode, pages[i - 1], 10251f934b00SYan, Zheng offset); 10265b64640cSYan, Zheng len = max(len, min_len); 1027e1966b49SYan, Zheng } 10285b64640cSYan, Zheng dout("writepages got pages at %llu~%llu\n", offset, len); 10291d3576fdSSage Weil 10305b64640cSYan, Zheng osd_req_op_extent_osd_data_pages(req, op_idx, data_pages, len, 1031a0102bdaSJeff Layton 0, from_pool, false); 10325b64640cSYan, Zheng osd_req_op_extent_update(req, op_idx, len); 1033e5975c7cSAlex Elder 10345b64640cSYan, Zheng BUG_ON(op_idx + 1 != req->r_num_ops); 10355b64640cSYan, Zheng 1036a0102bdaSJeff Layton from_pool = false; 10375b64640cSYan, Zheng if (i < locked_pages) { 10385b64640cSYan, Zheng BUG_ON(num_ops <= req->r_num_ops); 10395b64640cSYan, Zheng num_ops -= req->r_num_ops; 10405b64640cSYan, Zheng locked_pages -= i; 1041e5975c7cSAlex Elder 10425b64640cSYan, Zheng /* allocate new pages array for next request */ 10435b64640cSYan, Zheng data_pages = pages; 10446da2ec56SKees Cook pages = kmalloc_array(locked_pages, sizeof(*pages), 10455b64640cSYan, Zheng GFP_NOFS); 10465b64640cSYan, Zheng if (!pages) { 1047a0102bdaSJeff Layton from_pool = true; 1048a0102bdaSJeff Layton pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); 10495b64640cSYan, Zheng BUG_ON(!pages); 10505b64640cSYan, Zheng } 10515b64640cSYan, Zheng memcpy(pages, data_pages + i, 10525b64640cSYan, Zheng locked_pages * sizeof(*pages)); 10535b64640cSYan, Zheng memset(data_pages + i, 0, 10545b64640cSYan, Zheng locked_pages * sizeof(*pages)); 10555b64640cSYan, Zheng } else { 10565b64640cSYan, Zheng BUG_ON(num_ops != req->r_num_ops); 10575b64640cSYan, Zheng index = pages[i - 1]->index + 1; 10585b64640cSYan, Zheng /* request message now owns the pages array */ 10595b64640cSYan, Zheng pages = NULL; 10605b64640cSYan, Zheng } 1061e5975c7cSAlex Elder 1062fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 10639d6fcb08SSage Weil rc = ceph_osdc_start_request(&fsc->client->osdc, req, true); 10649d6fcb08SSage Weil BUG_ON(rc); 10651d3576fdSSage Weil req = NULL; 10661d3576fdSSage Weil 10675b64640cSYan, Zheng wbc->nr_to_write -= i; 10685b64640cSYan, Zheng if (pages) 10695b64640cSYan, Zheng goto new_request; 10705b64640cSYan, Zheng 10712a2d927eSYan, Zheng /* 10722a2d927eSYan, Zheng * We stop writing back only if we are not doing 10732a2d927eSYan, Zheng * integrity sync. In case of integrity sync we have to 10742a2d927eSYan, Zheng * keep going until we have written all the pages 10752a2d927eSYan, Zheng * we tagged for writeback prior to entering this loop. 10762a2d927eSYan, Zheng */ 10772a2d927eSYan, Zheng if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) 1078af9cc401SYan, Zheng done = true; 10791d3576fdSSage Weil 10801d3576fdSSage Weil release_pvec_pages: 10811d3576fdSSage Weil dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, 10821d3576fdSSage Weil pvec.nr ? pvec.pages[0] : NULL); 10831d3576fdSSage Weil pagevec_release(&pvec); 10841d3576fdSSage Weil } 10851d3576fdSSage Weil 10861d3576fdSSage Weil if (should_loop && !done) { 10871d3576fdSSage Weil /* more to do; loop back to beginning of file */ 10881d3576fdSSage Weil dout("writepages looping back to beginning of file\n"); 10892a2d927eSYan, Zheng end = start_index - 1; /* OK even when start_index == 0 */ 1090f275635eSYan, Zheng 1091f275635eSYan, Zheng /* to write dirty pages associated with next snapc, 1092f275635eSYan, Zheng * we need to wait until current writes complete */ 1093f275635eSYan, Zheng if (wbc->sync_mode != WB_SYNC_NONE && 1094f275635eSYan, Zheng start_index == 0 && /* all dirty pages were checked */ 1095f275635eSYan, Zheng !ceph_wbc.head_snapc) { 1096f275635eSYan, Zheng struct page *page; 1097f275635eSYan, Zheng unsigned i, nr; 1098f275635eSYan, Zheng index = 0; 1099f275635eSYan, Zheng while ((index <= end) && 1100f275635eSYan, Zheng (nr = pagevec_lookup_tag(&pvec, mapping, &index, 110167fd707fSJan Kara PAGECACHE_TAG_WRITEBACK))) { 1102f275635eSYan, Zheng for (i = 0; i < nr; i++) { 1103f275635eSYan, Zheng page = pvec.pages[i]; 1104f275635eSYan, Zheng if (page_snap_context(page) != snapc) 1105f275635eSYan, Zheng continue; 1106f275635eSYan, Zheng wait_on_page_writeback(page); 1107f275635eSYan, Zheng } 1108f275635eSYan, Zheng pagevec_release(&pvec); 1109f275635eSYan, Zheng cond_resched(); 1110f275635eSYan, Zheng } 1111f275635eSYan, Zheng } 1112f275635eSYan, Zheng 11132a2d927eSYan, Zheng start_index = 0; 11141d3576fdSSage Weil index = 0; 11151d3576fdSSage Weil goto retry; 11161d3576fdSSage Weil } 11171d3576fdSSage Weil 11181d3576fdSSage Weil if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 11191d3576fdSSage Weil mapping->writeback_index = index; 11201d3576fdSSage Weil 11211d3576fdSSage Weil out: 11221d3576fdSSage Weil ceph_osdc_put_request(req); 11232a2d927eSYan, Zheng ceph_put_snap_context(last_snapc); 11242a2d927eSYan, Zheng dout("writepages dend - startone, rc = %d\n", rc); 11251d3576fdSSage Weil return rc; 11261d3576fdSSage Weil } 11271d3576fdSSage Weil 11281d3576fdSSage Weil 11291d3576fdSSage Weil 11301d3576fdSSage Weil /* 11311d3576fdSSage Weil * See if a given @snapc is either writeable, or already written. 11321d3576fdSSage Weil */ 11331d3576fdSSage Weil static int context_is_writeable_or_written(struct inode *inode, 11341d3576fdSSage Weil struct ceph_snap_context *snapc) 11351d3576fdSSage Weil { 113605455e11SYan, Zheng struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL); 11376298a337SSage Weil int ret = !oldest || snapc->seq <= oldest->seq; 11386298a337SSage Weil 11396298a337SSage Weil ceph_put_snap_context(oldest); 11406298a337SSage Weil return ret; 11411d3576fdSSage Weil } 11421d3576fdSSage Weil 114318d620f0SJeff Layton /** 114418d620f0SJeff Layton * ceph_find_incompatible - find an incompatible context and return it 114518d620f0SJeff Layton * @page: page being dirtied 114618d620f0SJeff Layton * 114718d620f0SJeff Layton * We are only allowed to write into/dirty a page if the page is 114818d620f0SJeff Layton * clean, or already dirty within the same snap context. Returns a 114918d620f0SJeff Layton * conflicting context if there is one, NULL if there isn't, or a 115018d620f0SJeff Layton * negative error code on other errors. 115118d620f0SJeff Layton * 115218d620f0SJeff Layton * Must be called with page lock held. 115318d620f0SJeff Layton */ 115418d620f0SJeff Layton static struct ceph_snap_context * 1155d45156bfSJeff Layton ceph_find_incompatible(struct page *page) 115618d620f0SJeff Layton { 1157d45156bfSJeff Layton struct inode *inode = page->mapping->host; 115818d620f0SJeff Layton struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 115918d620f0SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 116018d620f0SJeff Layton 116150c9132dSJeff Layton if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) { 116218d620f0SJeff Layton dout(" page %p forced umount\n", page); 116318d620f0SJeff Layton return ERR_PTR(-EIO); 116418d620f0SJeff Layton } 116518d620f0SJeff Layton 116618d620f0SJeff Layton for (;;) { 116718d620f0SJeff Layton struct ceph_snap_context *snapc, *oldest; 116818d620f0SJeff Layton 116918d620f0SJeff Layton wait_on_page_writeback(page); 117018d620f0SJeff Layton 117118d620f0SJeff Layton snapc = page_snap_context(page); 117218d620f0SJeff Layton if (!snapc || snapc == ci->i_head_snapc) 117318d620f0SJeff Layton break; 117418d620f0SJeff Layton 117518d620f0SJeff Layton /* 117618d620f0SJeff Layton * this page is already dirty in another (older) snap 117718d620f0SJeff Layton * context! is it writeable now? 117818d620f0SJeff Layton */ 117918d620f0SJeff Layton oldest = get_oldest_context(inode, NULL, NULL); 118018d620f0SJeff Layton if (snapc->seq > oldest->seq) { 118118d620f0SJeff Layton /* not writeable -- return it for the caller to deal with */ 118218d620f0SJeff Layton ceph_put_snap_context(oldest); 118318d620f0SJeff Layton dout(" page %p snapc %p not current or oldest\n", page, snapc); 118418d620f0SJeff Layton return ceph_get_snap_context(snapc); 118518d620f0SJeff Layton } 118618d620f0SJeff Layton ceph_put_snap_context(oldest); 118718d620f0SJeff Layton 118818d620f0SJeff Layton /* yay, writeable, do it now (without dropping page lock) */ 118918d620f0SJeff Layton dout(" page %p snapc %p not current, but oldest\n", page, snapc); 119018d620f0SJeff Layton if (clear_page_dirty_for_io(page)) { 119118d620f0SJeff Layton int r = writepage_nounlock(page, NULL); 119218d620f0SJeff Layton if (r < 0) 119318d620f0SJeff Layton return ERR_PTR(r); 119418d620f0SJeff Layton } 119518d620f0SJeff Layton } 119618d620f0SJeff Layton return NULL; 119718d620f0SJeff Layton } 119818d620f0SJeff Layton 1199d801327dSJeff Layton static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, 1200d801327dSJeff Layton struct page *page, void **_fsdata) 1201d801327dSJeff Layton { 1202d801327dSJeff Layton struct inode *inode = file_inode(file); 1203d801327dSJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 1204d801327dSJeff Layton struct ceph_snap_context *snapc; 1205d801327dSJeff Layton 1206d801327dSJeff Layton snapc = ceph_find_incompatible(page); 1207d801327dSJeff Layton if (snapc) { 1208d801327dSJeff Layton int r; 1209d801327dSJeff Layton 1210d801327dSJeff Layton unlock_page(page); 1211d801327dSJeff Layton put_page(page); 1212d801327dSJeff Layton if (IS_ERR(snapc)) 1213d801327dSJeff Layton return PTR_ERR(snapc); 1214d801327dSJeff Layton 1215d801327dSJeff Layton ceph_queue_writeback(inode); 1216d801327dSJeff Layton r = wait_event_killable(ci->i_cap_wq, 1217d801327dSJeff Layton context_is_writeable_or_written(inode, snapc)); 1218d801327dSJeff Layton ceph_put_snap_context(snapc); 1219d801327dSJeff Layton return r == 0 ? -EAGAIN : r; 1220d801327dSJeff Layton } 1221d801327dSJeff Layton return 0; 1222d801327dSJeff Layton } 1223d801327dSJeff Layton 12241d3576fdSSage Weil /* 12251d3576fdSSage Weil * We are only allowed to write into/dirty the page if the page is 12261d3576fdSSage Weil * clean, or already dirty within the same snap context. 12274af6b225SYehuda Sadeh */ 12284af6b225SYehuda Sadeh static int ceph_write_begin(struct file *file, struct address_space *mapping, 12294af6b225SYehuda Sadeh loff_t pos, unsigned len, unsigned flags, 12304af6b225SYehuda Sadeh struct page **pagep, void **fsdata) 12314af6b225SYehuda Sadeh { 1232496ad9aaSAl Viro struct inode *inode = file_inode(file); 12331cc16990SJeff Layton struct ceph_inode_info *ci = ceph_inode(inode); 12341cc16990SJeff Layton struct page *page = NULL; 123509cbfeafSKirill A. Shutemov pgoff_t index = pos >> PAGE_SHIFT; 1236d801327dSJeff Layton int r; 12374af6b225SYehuda Sadeh 1238d801327dSJeff Layton /* 1239d801327dSJeff Layton * Uninlining should have already been done and everything updated, EXCEPT 1240d801327dSJeff Layton * for inline_version sent to the MDS. 1241d801327dSJeff Layton */ 1242d801327dSJeff Layton if (ci->i_inline_version != CEPH_INLINE_NONE) { 12434a357f50SJeff Layton page = grab_cache_page_write_begin(mapping, index, flags); 1244d801327dSJeff Layton if (!page) 1245d801327dSJeff Layton return -ENOMEM; 12461cc16990SJeff Layton 12471cc16990SJeff Layton /* 1248d801327dSJeff Layton * The inline_version on a new inode is set to 1. If that's the 1249d801327dSJeff Layton * case, then the page is brand new and isn't yet Uptodate. 12501cc16990SJeff Layton */ 1251d801327dSJeff Layton r = 0; 1252d801327dSJeff Layton if (index == 0 && ci->i_inline_version != 1) { 1253d801327dSJeff Layton if (!PageUptodate(page)) { 1254d801327dSJeff Layton WARN_ONCE(1, "ceph: write_begin called on still-inlined inode (inline_version %llu)!\n", 1255d801327dSJeff Layton ci->i_inline_version); 1256d801327dSJeff Layton r = -EINVAL; 1257d801327dSJeff Layton } 1258d801327dSJeff Layton goto out; 1259d801327dSJeff Layton } 1260d801327dSJeff Layton zero_user_segment(page, 0, PAGE_SIZE); 1261d801327dSJeff Layton SetPageUptodate(page); 1262d801327dSJeff Layton goto out; 12631cc16990SJeff Layton } 12641cc16990SJeff Layton 1265d801327dSJeff Layton r = netfs_write_begin(file, inode->i_mapping, pos, len, 0, &page, NULL, 1266d801327dSJeff Layton &ceph_netfs_read_ops, NULL); 1267d801327dSJeff Layton out: 1268d801327dSJeff Layton if (r == 0) 1269d801327dSJeff Layton wait_on_page_fscache(page); 12701cc16990SJeff Layton if (r < 0) { 1271d801327dSJeff Layton if (page) 12721cc16990SJeff Layton put_page(page); 12731cc16990SJeff Layton } else { 1274d801327dSJeff Layton WARN_ON_ONCE(!PageLocked(page)); 12751cc16990SJeff Layton *pagep = page; 12761cc16990SJeff Layton } 12774af6b225SYehuda Sadeh return r; 12784af6b225SYehuda Sadeh } 12794af6b225SYehuda Sadeh 12804af6b225SYehuda Sadeh /* 12811d3576fdSSage Weil * we don't do anything in here that simple_write_end doesn't do 12825dda377cSYan, Zheng * except adjust dirty page accounting 12831d3576fdSSage Weil */ 12841d3576fdSSage Weil static int ceph_write_end(struct file *file, struct address_space *mapping, 12851d3576fdSSage Weil loff_t pos, unsigned len, unsigned copied, 12861d3576fdSSage Weil struct page *page, void *fsdata) 12871d3576fdSSage Weil { 1288496ad9aaSAl Viro struct inode *inode = file_inode(file); 1289efb0ca76SYan, Zheng bool check_cap = false; 12901d3576fdSSage Weil 12911d3576fdSSage Weil dout("write_end file %p inode %p page %p %d~%d (%d)\n", file, 12921d3576fdSSage Weil inode, page, (int)pos, (int)copied, (int)len); 12931d3576fdSSage Weil 12941d3576fdSSage Weil /* zero the stale part of the page if we did a short copy */ 1295b9de313cSAl Viro if (!PageUptodate(page)) { 1296b9de313cSAl Viro if (copied < len) { 1297b9de313cSAl Viro copied = 0; 1298b9de313cSAl Viro goto out; 1299b9de313cSAl Viro } 1300b9de313cSAl Viro SetPageUptodate(page); 1301b9de313cSAl Viro } 13021d3576fdSSage Weil 13031d3576fdSSage Weil /* did file size increase? */ 130499c88e69SYan, Zheng if (pos+copied > i_size_read(inode)) 13051d3576fdSSage Weil check_cap = ceph_inode_set_size(inode, pos+copied); 13061d3576fdSSage Weil 13071d3576fdSSage Weil set_page_dirty(page); 13081d3576fdSSage Weil 1309b9de313cSAl Viro out: 13101d3576fdSSage Weil unlock_page(page); 131109cbfeafSKirill A. Shutemov put_page(page); 13121d3576fdSSage Weil 13131d3576fdSSage Weil if (check_cap) 13141d3576fdSSage Weil ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY, NULL); 13151d3576fdSSage Weil 13161d3576fdSSage Weil return copied; 13171d3576fdSSage Weil } 13181d3576fdSSage Weil 13191d3576fdSSage Weil /* 13201d3576fdSSage Weil * we set .direct_IO to indicate direct io is supported, but since we 13211d3576fdSSage Weil * intercept O_DIRECT reads and writes early, this function should 13221d3576fdSSage Weil * never get called. 13231d3576fdSSage Weil */ 1324c8b8e32dSChristoph Hellwig static ssize_t ceph_direct_io(struct kiocb *iocb, struct iov_iter *iter) 13251d3576fdSSage Weil { 13261d3576fdSSage Weil WARN_ON(1); 13271d3576fdSSage Weil return -EINVAL; 13281d3576fdSSage Weil } 13291d3576fdSSage Weil 13301d3576fdSSage Weil const struct address_space_operations ceph_aops = { 13311d3576fdSSage Weil .readpage = ceph_readpage, 1332*49870056SJeff Layton .readahead = ceph_readahead, 13331d3576fdSSage Weil .writepage = ceph_writepage, 13341d3576fdSSage Weil .writepages = ceph_writepages_start, 13351d3576fdSSage Weil .write_begin = ceph_write_begin, 13361d3576fdSSage Weil .write_end = ceph_write_end, 13371d3576fdSSage Weil .set_page_dirty = ceph_set_page_dirty, 13381d3576fdSSage Weil .invalidatepage = ceph_invalidatepage, 13391d3576fdSSage Weil .releasepage = ceph_releasepage, 13401d3576fdSSage Weil .direct_IO = ceph_direct_io, 13411d3576fdSSage Weil }; 13421d3576fdSSage Weil 13434f7e89f6SYan, Zheng static void ceph_block_sigs(sigset_t *oldset) 13444f7e89f6SYan, Zheng { 13454f7e89f6SYan, Zheng sigset_t mask; 13464f7e89f6SYan, Zheng siginitsetinv(&mask, sigmask(SIGKILL)); 13474f7e89f6SYan, Zheng sigprocmask(SIG_BLOCK, &mask, oldset); 13484f7e89f6SYan, Zheng } 13494f7e89f6SYan, Zheng 13504f7e89f6SYan, Zheng static void ceph_restore_sigs(sigset_t *oldset) 13514f7e89f6SYan, Zheng { 13524f7e89f6SYan, Zheng sigprocmask(SIG_SETMASK, oldset, NULL); 13534f7e89f6SYan, Zheng } 13541d3576fdSSage Weil 13551d3576fdSSage Weil /* 13561d3576fdSSage Weil * vm ops 13571d3576fdSSage Weil */ 135824499847SSouptick Joarder static vm_fault_t ceph_filemap_fault(struct vm_fault *vmf) 135961f68816SYan, Zheng { 136011bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 136161f68816SYan, Zheng struct inode *inode = file_inode(vma->vm_file); 136261f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 136361f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 13643738daa6SYan, Zheng struct page *pinned_page = NULL; 1365c403c3a2SMatthew Wilcox (Oracle) loff_t off = (loff_t)vmf->pgoff << PAGE_SHIFT; 136624499847SSouptick Joarder int want, got, err; 13674f7e89f6SYan, Zheng sigset_t oldset; 136824499847SSouptick Joarder vm_fault_t ret = VM_FAULT_SIGBUS; 13694f7e89f6SYan, Zheng 13704f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 137161f68816SYan, Zheng 137261f68816SYan, Zheng dout("filemap_fault %p %llx.%llx %llu~%zd trying to get caps\n", 137309cbfeafSKirill A. Shutemov inode, ceph_vinop(inode), off, (size_t)PAGE_SIZE); 137461f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 137561f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; 137661f68816SYan, Zheng else 137761f68816SYan, Zheng want = CEPH_CAP_FILE_CACHE; 13784f7e89f6SYan, Zheng 137961f68816SYan, Zheng got = 0; 13805e3ded1bSYan, Zheng err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_RD, want, -1, 13815e3ded1bSYan, Zheng &got, &pinned_page); 138224499847SSouptick Joarder if (err < 0) 13834f7e89f6SYan, Zheng goto out_restore; 13846ce026e4SYan, Zheng 138561f68816SYan, Zheng dout("filemap_fault %p %llu~%zd got cap refs on %s\n", 138609cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ceph_cap_string(got)); 138761f68816SYan, Zheng 138883701246SYan, Zheng if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || 13892b1ac852SYan, Zheng ci->i_inline_version == CEPH_INLINE_NONE) { 13905d988308SYan, Zheng CEPH_DEFINE_RW_CONTEXT(rw_ctx, got); 13915d988308SYan, Zheng ceph_add_rw_context(fi, &rw_ctx); 139211bac800SDave Jiang ret = filemap_fault(vmf); 13935d988308SYan, Zheng ceph_del_rw_context(fi, &rw_ctx); 139424499847SSouptick Joarder dout("filemap_fault %p %llu~%zd drop cap refs %s ret %x\n", 139524499847SSouptick Joarder inode, off, (size_t)PAGE_SIZE, 139624499847SSouptick Joarder ceph_cap_string(got), ret); 13972b1ac852SYan, Zheng } else 139824499847SSouptick Joarder err = -EAGAIN; 139961f68816SYan, Zheng 14003738daa6SYan, Zheng if (pinned_page) 140109cbfeafSKirill A. Shutemov put_page(pinned_page); 140261f68816SYan, Zheng ceph_put_cap_refs(ci, got); 140361f68816SYan, Zheng 140424499847SSouptick Joarder if (err != -EAGAIN) 14054f7e89f6SYan, Zheng goto out_restore; 140683701246SYan, Zheng 140783701246SYan, Zheng /* read inline data */ 140809cbfeafSKirill A. Shutemov if (off >= PAGE_SIZE) { 140983701246SYan, Zheng /* does not support inline data > PAGE_SIZE */ 141083701246SYan, Zheng ret = VM_FAULT_SIGBUS; 141183701246SYan, Zheng } else { 141283701246SYan, Zheng struct address_space *mapping = inode->i_mapping; 141383701246SYan, Zheng struct page *page = find_or_create_page(mapping, 0, 1414c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1415c62d2555SMichal Hocko ~__GFP_FS)); 141683701246SYan, Zheng if (!page) { 141783701246SYan, Zheng ret = VM_FAULT_OOM; 14184f7e89f6SYan, Zheng goto out_inline; 141983701246SYan, Zheng } 142024499847SSouptick Joarder err = __ceph_do_getattr(inode, page, 142183701246SYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 142224499847SSouptick Joarder if (err < 0 || off >= i_size_read(inode)) { 142383701246SYan, Zheng unlock_page(page); 142409cbfeafSKirill A. Shutemov put_page(page); 1425c64a2b05SSouptick Joarder ret = vmf_error(err); 14264f7e89f6SYan, Zheng goto out_inline; 142783701246SYan, Zheng } 142824499847SSouptick Joarder if (err < PAGE_SIZE) 142924499847SSouptick Joarder zero_user_segment(page, err, PAGE_SIZE); 143083701246SYan, Zheng else 143183701246SYan, Zheng flush_dcache_page(page); 143283701246SYan, Zheng SetPageUptodate(page); 143383701246SYan, Zheng vmf->page = page; 143483701246SYan, Zheng ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; 14354f7e89f6SYan, Zheng out_inline: 143624499847SSouptick Joarder dout("filemap_fault %p %llu~%zd read inline data ret %x\n", 143709cbfeafSKirill A. Shutemov inode, off, (size_t)PAGE_SIZE, ret); 14384f7e89f6SYan, Zheng } 14394f7e89f6SYan, Zheng out_restore: 14404f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 144124499847SSouptick Joarder if (err < 0) 144224499847SSouptick Joarder ret = vmf_error(err); 14436ce026e4SYan, Zheng 144461f68816SYan, Zheng return ret; 144561f68816SYan, Zheng } 14461d3576fdSSage Weil 144724499847SSouptick Joarder static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) 14481d3576fdSSage Weil { 144911bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 1450496ad9aaSAl Viro struct inode *inode = file_inode(vma->vm_file); 145161f68816SYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 145261f68816SYan, Zheng struct ceph_file_info *fi = vma->vm_file->private_data; 1453f66fd9f0SYan, Zheng struct ceph_cap_flush *prealloc_cf; 145461f68816SYan, Zheng struct page *page = vmf->page; 14556285bc23SAlex Elder loff_t off = page_offset(page); 145661f68816SYan, Zheng loff_t size = i_size_read(inode); 145761f68816SYan, Zheng size_t len; 145824499847SSouptick Joarder int want, got, err; 14594f7e89f6SYan, Zheng sigset_t oldset; 146024499847SSouptick Joarder vm_fault_t ret = VM_FAULT_SIGBUS; 14611d3576fdSSage Weil 1462f66fd9f0SYan, Zheng prealloc_cf = ceph_alloc_cap_flush(); 1463f66fd9f0SYan, Zheng if (!prealloc_cf) 14646ce026e4SYan, Zheng return VM_FAULT_OOM; 1465f66fd9f0SYan, Zheng 1466249c1df5SJeff Layton sb_start_pagefault(inode->i_sb); 14674f7e89f6SYan, Zheng ceph_block_sigs(&oldset); 14681d3576fdSSage Weil 146928127bddSYan, Zheng if (ci->i_inline_version != CEPH_INLINE_NONE) { 147028127bddSYan, Zheng struct page *locked_page = NULL; 147128127bddSYan, Zheng if (off == 0) { 147228127bddSYan, Zheng lock_page(page); 147328127bddSYan, Zheng locked_page = page; 147428127bddSYan, Zheng } 147524499847SSouptick Joarder err = ceph_uninline_data(vma->vm_file, locked_page); 147628127bddSYan, Zheng if (locked_page) 147728127bddSYan, Zheng unlock_page(locked_page); 147824499847SSouptick Joarder if (err < 0) 1479f66fd9f0SYan, Zheng goto out_free; 1480f66fd9f0SYan, Zheng } 148128127bddSYan, Zheng 148209cbfeafSKirill A. Shutemov if (off + PAGE_SIZE <= size) 148309cbfeafSKirill A. Shutemov len = PAGE_SIZE; 14841d3576fdSSage Weil else 148509cbfeafSKirill A. Shutemov len = size & ~PAGE_MASK; 14861d3576fdSSage Weil 148761f68816SYan, Zheng dout("page_mkwrite %p %llx.%llx %llu~%zd getting caps i_size %llu\n", 148861f68816SYan, Zheng inode, ceph_vinop(inode), off, len, size); 148961f68816SYan, Zheng if (fi->fmode & CEPH_FILE_MODE_LAZY) 149061f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; 149161f68816SYan, Zheng else 149261f68816SYan, Zheng want = CEPH_CAP_FILE_BUFFER; 14934f7e89f6SYan, Zheng 149461f68816SYan, Zheng got = 0; 14955e3ded1bSYan, Zheng err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_WR, want, off + len, 14963738daa6SYan, Zheng &got, NULL); 149724499847SSouptick Joarder if (err < 0) 1498f66fd9f0SYan, Zheng goto out_free; 14996ce026e4SYan, Zheng 150061f68816SYan, Zheng dout("page_mkwrite %p %llu~%zd got cap refs on %s\n", 150161f68816SYan, Zheng inode, off, len, ceph_cap_string(got)); 150261f68816SYan, Zheng 150361f68816SYan, Zheng /* Update time before taking page lock */ 150461f68816SYan, Zheng file_update_time(vma->vm_file); 15055c308356SJeff Layton inode_inc_iversion_raw(inode); 15064af6b225SYehuda Sadeh 1507f0b33df5SYan, Zheng do { 1508d45156bfSJeff Layton struct ceph_snap_context *snapc; 1509d45156bfSJeff Layton 15104af6b225SYehuda Sadeh lock_page(page); 15114af6b225SYehuda Sadeh 1512cb03c143SAndreas Gruenbacher if (page_mkwrite_check_truncate(page, inode) < 0) { 1513f9cac5acSYan, Zheng unlock_page(page); 15146ce026e4SYan, Zheng ret = VM_FAULT_NOPAGE; 1515f0b33df5SYan, Zheng break; 1516f9cac5acSYan, Zheng } 15174af6b225SYehuda Sadeh 1518d45156bfSJeff Layton snapc = ceph_find_incompatible(page); 1519d45156bfSJeff Layton if (!snapc) { 15204af6b225SYehuda Sadeh /* success. we'll keep the page locked. */ 15211d3576fdSSage Weil set_page_dirty(page); 15221d3576fdSSage Weil ret = VM_FAULT_LOCKED; 1523d45156bfSJeff Layton break; 15241d3576fdSSage Weil } 1525d45156bfSJeff Layton 1526d45156bfSJeff Layton unlock_page(page); 1527d45156bfSJeff Layton 1528d45156bfSJeff Layton if (IS_ERR(snapc)) { 1529d45156bfSJeff Layton ret = VM_FAULT_SIGBUS; 1530d45156bfSJeff Layton break; 1531d45156bfSJeff Layton } 1532d45156bfSJeff Layton 1533d45156bfSJeff Layton ceph_queue_writeback(inode); 1534d45156bfSJeff Layton err = wait_event_killable(ci->i_cap_wq, 1535d45156bfSJeff Layton context_is_writeable_or_written(inode, snapc)); 1536d45156bfSJeff Layton ceph_put_snap_context(snapc); 1537d45156bfSJeff Layton } while (err == 0); 1538f0b33df5SYan, Zheng 153928127bddSYan, Zheng if (ret == VM_FAULT_LOCKED || 154028127bddSYan, Zheng ci->i_inline_version != CEPH_INLINE_NONE) { 154161f68816SYan, Zheng int dirty; 154261f68816SYan, Zheng spin_lock(&ci->i_ceph_lock); 154328127bddSYan, Zheng ci->i_inline_version = CEPH_INLINE_NONE; 1544f66fd9f0SYan, Zheng dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, 1545f66fd9f0SYan, Zheng &prealloc_cf); 154661f68816SYan, Zheng spin_unlock(&ci->i_ceph_lock); 154761f68816SYan, Zheng if (dirty) 154861f68816SYan, Zheng __mark_inode_dirty(inode, dirty); 154961f68816SYan, Zheng } 155061f68816SYan, Zheng 155124499847SSouptick Joarder dout("page_mkwrite %p %llu~%zd dropping cap refs on %s ret %x\n", 155261f68816SYan, Zheng inode, off, len, ceph_cap_string(got), ret); 1553a8810cdcSJeff Layton ceph_put_cap_refs_async(ci, got); 1554f66fd9f0SYan, Zheng out_free: 15554f7e89f6SYan, Zheng ceph_restore_sigs(&oldset); 1556249c1df5SJeff Layton sb_end_pagefault(inode->i_sb); 1557f66fd9f0SYan, Zheng ceph_free_cap_flush(prealloc_cf); 155824499847SSouptick Joarder if (err < 0) 155924499847SSouptick Joarder ret = vmf_error(err); 15601d3576fdSSage Weil return ret; 15611d3576fdSSage Weil } 15621d3576fdSSage Weil 156331c542a1SYan, Zheng void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, 156431c542a1SYan, Zheng char *data, size_t len) 156531c542a1SYan, Zheng { 156631c542a1SYan, Zheng struct address_space *mapping = inode->i_mapping; 156731c542a1SYan, Zheng struct page *page; 156831c542a1SYan, Zheng 156931c542a1SYan, Zheng if (locked_page) { 157031c542a1SYan, Zheng page = locked_page; 157131c542a1SYan, Zheng } else { 157231c542a1SYan, Zheng if (i_size_read(inode) == 0) 157331c542a1SYan, Zheng return; 157431c542a1SYan, Zheng page = find_or_create_page(mapping, 0, 1575c62d2555SMichal Hocko mapping_gfp_constraint(mapping, 1576c62d2555SMichal Hocko ~__GFP_FS)); 157731c542a1SYan, Zheng if (!page) 157831c542a1SYan, Zheng return; 157931c542a1SYan, Zheng if (PageUptodate(page)) { 158031c542a1SYan, Zheng unlock_page(page); 158109cbfeafSKirill A. Shutemov put_page(page); 158231c542a1SYan, Zheng return; 158331c542a1SYan, Zheng } 158431c542a1SYan, Zheng } 158531c542a1SYan, Zheng 15860668ff52SIlya Dryomov dout("fill_inline_data %p %llx.%llx len %zu locked_page %p\n", 158731c542a1SYan, Zheng inode, ceph_vinop(inode), len, locked_page); 158831c542a1SYan, Zheng 158931c542a1SYan, Zheng if (len > 0) { 159031c542a1SYan, Zheng void *kaddr = kmap_atomic(page); 159131c542a1SYan, Zheng memcpy(kaddr, data, len); 159231c542a1SYan, Zheng kunmap_atomic(kaddr); 159331c542a1SYan, Zheng } 159431c542a1SYan, Zheng 159531c542a1SYan, Zheng if (page != locked_page) { 159609cbfeafSKirill A. Shutemov if (len < PAGE_SIZE) 159709cbfeafSKirill A. Shutemov zero_user_segment(page, len, PAGE_SIZE); 159831c542a1SYan, Zheng else 159931c542a1SYan, Zheng flush_dcache_page(page); 160031c542a1SYan, Zheng 160131c542a1SYan, Zheng SetPageUptodate(page); 160231c542a1SYan, Zheng unlock_page(page); 160309cbfeafSKirill A. Shutemov put_page(page); 160431c542a1SYan, Zheng } 160531c542a1SYan, Zheng } 160631c542a1SYan, Zheng 160728127bddSYan, Zheng int ceph_uninline_data(struct file *filp, struct page *locked_page) 160828127bddSYan, Zheng { 160928127bddSYan, Zheng struct inode *inode = file_inode(filp); 161028127bddSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 161128127bddSYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(inode); 161228127bddSYan, Zheng struct ceph_osd_request *req; 161328127bddSYan, Zheng struct page *page = NULL; 161428127bddSYan, Zheng u64 len, inline_version; 161528127bddSYan, Zheng int err = 0; 161628127bddSYan, Zheng bool from_pagecache = false; 161728127bddSYan, Zheng 161828127bddSYan, Zheng spin_lock(&ci->i_ceph_lock); 161928127bddSYan, Zheng inline_version = ci->i_inline_version; 162028127bddSYan, Zheng spin_unlock(&ci->i_ceph_lock); 162128127bddSYan, Zheng 162228127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu\n", 162328127bddSYan, Zheng inode, ceph_vinop(inode), inline_version); 162428127bddSYan, Zheng 162528127bddSYan, Zheng if (inline_version == 1 || /* initial version, no data */ 162628127bddSYan, Zheng inline_version == CEPH_INLINE_NONE) 162728127bddSYan, Zheng goto out; 162828127bddSYan, Zheng 162928127bddSYan, Zheng if (locked_page) { 163028127bddSYan, Zheng page = locked_page; 163128127bddSYan, Zheng WARN_ON(!PageUptodate(page)); 163228127bddSYan, Zheng } else if (ceph_caps_issued(ci) & 163328127bddSYan, Zheng (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) { 163428127bddSYan, Zheng page = find_get_page(inode->i_mapping, 0); 163528127bddSYan, Zheng if (page) { 163628127bddSYan, Zheng if (PageUptodate(page)) { 163728127bddSYan, Zheng from_pagecache = true; 163828127bddSYan, Zheng lock_page(page); 163928127bddSYan, Zheng } else { 164009cbfeafSKirill A. Shutemov put_page(page); 164128127bddSYan, Zheng page = NULL; 164228127bddSYan, Zheng } 164328127bddSYan, Zheng } 164428127bddSYan, Zheng } 164528127bddSYan, Zheng 164628127bddSYan, Zheng if (page) { 164728127bddSYan, Zheng len = i_size_read(inode); 164809cbfeafSKirill A. Shutemov if (len > PAGE_SIZE) 164909cbfeafSKirill A. Shutemov len = PAGE_SIZE; 165028127bddSYan, Zheng } else { 165128127bddSYan, Zheng page = __page_cache_alloc(GFP_NOFS); 165228127bddSYan, Zheng if (!page) { 165328127bddSYan, Zheng err = -ENOMEM; 165428127bddSYan, Zheng goto out; 165528127bddSYan, Zheng } 165628127bddSYan, Zheng err = __ceph_do_getattr(inode, page, 165728127bddSYan, Zheng CEPH_STAT_CAP_INLINE_DATA, true); 165828127bddSYan, Zheng if (err < 0) { 165928127bddSYan, Zheng /* no inline data */ 166028127bddSYan, Zheng if (err == -ENODATA) 166128127bddSYan, Zheng err = 0; 166228127bddSYan, Zheng goto out; 166328127bddSYan, Zheng } 166428127bddSYan, Zheng len = err; 166528127bddSYan, Zheng } 166628127bddSYan, Zheng 166728127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 166828127bddSYan, Zheng ceph_vino(inode), 0, &len, 0, 1, 166954ea0046SIlya Dryomov CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, 167034b759b4SIlya Dryomov NULL, 0, 0, false); 167128127bddSYan, Zheng if (IS_ERR(req)) { 167228127bddSYan, Zheng err = PTR_ERR(req); 167328127bddSYan, Zheng goto out; 167428127bddSYan, Zheng } 167528127bddSYan, Zheng 1676fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 167728127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 167828127bddSYan, Zheng if (!err) 167928127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 168028127bddSYan, Zheng ceph_osdc_put_request(req); 168128127bddSYan, Zheng if (err < 0) 168228127bddSYan, Zheng goto out; 168328127bddSYan, Zheng 168428127bddSYan, Zheng req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, 168528127bddSYan, Zheng ceph_vino(inode), 0, &len, 1, 3, 168654ea0046SIlya Dryomov CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, 168734b759b4SIlya Dryomov NULL, ci->i_truncate_seq, 168834b759b4SIlya Dryomov ci->i_truncate_size, false); 168928127bddSYan, Zheng if (IS_ERR(req)) { 169028127bddSYan, Zheng err = PTR_ERR(req); 169128127bddSYan, Zheng goto out; 169228127bddSYan, Zheng } 169328127bddSYan, Zheng 169428127bddSYan, Zheng osd_req_op_extent_osd_data_pages(req, 1, &page, len, 0, false, false); 169528127bddSYan, Zheng 1696ec137c10SYan, Zheng { 1697ec137c10SYan, Zheng __le64 xattr_buf = cpu_to_le64(inline_version); 169828127bddSYan, Zheng err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, 1699ec137c10SYan, Zheng "inline_version", &xattr_buf, 1700ec137c10SYan, Zheng sizeof(xattr_buf), 170128127bddSYan, Zheng CEPH_OSD_CMPXATTR_OP_GT, 170228127bddSYan, Zheng CEPH_OSD_CMPXATTR_MODE_U64); 170328127bddSYan, Zheng if (err) 170428127bddSYan, Zheng goto out_put; 1705ec137c10SYan, Zheng } 170628127bddSYan, Zheng 1707ec137c10SYan, Zheng { 1708ec137c10SYan, Zheng char xattr_buf[32]; 1709ec137c10SYan, Zheng int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), 1710ec137c10SYan, Zheng "%llu", inline_version); 171128127bddSYan, Zheng err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, 1712ec137c10SYan, Zheng "inline_version", 1713ec137c10SYan, Zheng xattr_buf, xattr_len, 0, 0); 171428127bddSYan, Zheng if (err) 171528127bddSYan, Zheng goto out_put; 1716ec137c10SYan, Zheng } 171728127bddSYan, Zheng 1718fac02ddfSArnd Bergmann req->r_mtime = inode->i_mtime; 171928127bddSYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, req, false); 172028127bddSYan, Zheng if (!err) 172128127bddSYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, req); 172297e27aaaSXiubo Li 172397e27aaaSXiubo Li ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency, 172497e27aaaSXiubo Li req->r_end_latency, err); 172597e27aaaSXiubo Li 172628127bddSYan, Zheng out_put: 172728127bddSYan, Zheng ceph_osdc_put_request(req); 172828127bddSYan, Zheng if (err == -ECANCELED) 172928127bddSYan, Zheng err = 0; 173028127bddSYan, Zheng out: 173128127bddSYan, Zheng if (page && page != locked_page) { 173228127bddSYan, Zheng if (from_pagecache) { 173328127bddSYan, Zheng unlock_page(page); 173409cbfeafSKirill A. Shutemov put_page(page); 173528127bddSYan, Zheng } else 173628127bddSYan, Zheng __free_pages(page, 0); 173728127bddSYan, Zheng } 173828127bddSYan, Zheng 173928127bddSYan, Zheng dout("uninline_data %p %llx.%llx inline_version %llu = %d\n", 174028127bddSYan, Zheng inode, ceph_vinop(inode), inline_version, err); 174128127bddSYan, Zheng return err; 174228127bddSYan, Zheng } 174328127bddSYan, Zheng 17447cbea8dcSKirill A. Shutemov static const struct vm_operations_struct ceph_vmops = { 174561f68816SYan, Zheng .fault = ceph_filemap_fault, 17461d3576fdSSage Weil .page_mkwrite = ceph_page_mkwrite, 17471d3576fdSSage Weil }; 17481d3576fdSSage Weil 17491d3576fdSSage Weil int ceph_mmap(struct file *file, struct vm_area_struct *vma) 17501d3576fdSSage Weil { 17511d3576fdSSage Weil struct address_space *mapping = file->f_mapping; 17521d3576fdSSage Weil 17531d3576fdSSage Weil if (!mapping->a_ops->readpage) 17541d3576fdSSage Weil return -ENOEXEC; 17551d3576fdSSage Weil file_accessed(file); 17561d3576fdSSage Weil vma->vm_ops = &ceph_vmops; 17571d3576fdSSage Weil return 0; 17581d3576fdSSage Weil } 175910183a69SYan, Zheng 176010183a69SYan, Zheng enum { 176110183a69SYan, Zheng POOL_READ = 1, 176210183a69SYan, Zheng POOL_WRITE = 2, 176310183a69SYan, Zheng }; 176410183a69SYan, Zheng 1765779fe0fbSYan, Zheng static int __ceph_pool_perm_get(struct ceph_inode_info *ci, 1766779fe0fbSYan, Zheng s64 pool, struct ceph_string *pool_ns) 176710183a69SYan, Zheng { 176810183a69SYan, Zheng struct ceph_fs_client *fsc = ceph_inode_to_client(&ci->vfs_inode); 176910183a69SYan, Zheng struct ceph_mds_client *mdsc = fsc->mdsc; 177010183a69SYan, Zheng struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; 177110183a69SYan, Zheng struct rb_node **p, *parent; 177210183a69SYan, Zheng struct ceph_pool_perm *perm; 177310183a69SYan, Zheng struct page **pages; 1774779fe0fbSYan, Zheng size_t pool_ns_len; 177510183a69SYan, Zheng int err = 0, err2 = 0, have = 0; 177610183a69SYan, Zheng 177710183a69SYan, Zheng down_read(&mdsc->pool_perm_rwsem); 177810183a69SYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 177910183a69SYan, Zheng while (*p) { 178010183a69SYan, Zheng perm = rb_entry(*p, struct ceph_pool_perm, node); 178110183a69SYan, Zheng if (pool < perm->pool) 178210183a69SYan, Zheng p = &(*p)->rb_left; 178310183a69SYan, Zheng else if (pool > perm->pool) 178410183a69SYan, Zheng p = &(*p)->rb_right; 178510183a69SYan, Zheng else { 1786779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1787779fe0fbSYan, Zheng perm->pool_ns, 1788779fe0fbSYan, Zheng perm->pool_ns_len); 1789779fe0fbSYan, Zheng if (ret < 0) 1790779fe0fbSYan, Zheng p = &(*p)->rb_left; 1791779fe0fbSYan, Zheng else if (ret > 0) 1792779fe0fbSYan, Zheng p = &(*p)->rb_right; 1793779fe0fbSYan, Zheng else { 179410183a69SYan, Zheng have = perm->perm; 179510183a69SYan, Zheng break; 179610183a69SYan, Zheng } 179710183a69SYan, Zheng } 1798779fe0fbSYan, Zheng } 179910183a69SYan, Zheng up_read(&mdsc->pool_perm_rwsem); 180010183a69SYan, Zheng if (*p) 180110183a69SYan, Zheng goto out; 180210183a69SYan, Zheng 1803779fe0fbSYan, Zheng if (pool_ns) 1804779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s no perm cached\n", 1805779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str); 1806779fe0fbSYan, Zheng else 18077627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld no perm cached\n", pool); 180810183a69SYan, Zheng 180910183a69SYan, Zheng down_write(&mdsc->pool_perm_rwsem); 1810779fe0fbSYan, Zheng p = &mdsc->pool_perm_tree.rb_node; 181110183a69SYan, Zheng parent = NULL; 181210183a69SYan, Zheng while (*p) { 181310183a69SYan, Zheng parent = *p; 181410183a69SYan, Zheng perm = rb_entry(parent, struct ceph_pool_perm, node); 181510183a69SYan, Zheng if (pool < perm->pool) 181610183a69SYan, Zheng p = &(*p)->rb_left; 181710183a69SYan, Zheng else if (pool > perm->pool) 181810183a69SYan, Zheng p = &(*p)->rb_right; 181910183a69SYan, Zheng else { 1820779fe0fbSYan, Zheng int ret = ceph_compare_string(pool_ns, 1821779fe0fbSYan, Zheng perm->pool_ns, 1822779fe0fbSYan, Zheng perm->pool_ns_len); 1823779fe0fbSYan, Zheng if (ret < 0) 1824779fe0fbSYan, Zheng p = &(*p)->rb_left; 1825779fe0fbSYan, Zheng else if (ret > 0) 1826779fe0fbSYan, Zheng p = &(*p)->rb_right; 1827779fe0fbSYan, Zheng else { 182810183a69SYan, Zheng have = perm->perm; 182910183a69SYan, Zheng break; 183010183a69SYan, Zheng } 183110183a69SYan, Zheng } 1832779fe0fbSYan, Zheng } 183310183a69SYan, Zheng if (*p) { 183410183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 183510183a69SYan, Zheng goto out; 183610183a69SYan, Zheng } 183710183a69SYan, Zheng 183834b759b4SIlya Dryomov rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 183910183a69SYan, Zheng 1, false, GFP_NOFS); 184010183a69SYan, Zheng if (!rd_req) { 184110183a69SYan, Zheng err = -ENOMEM; 184210183a69SYan, Zheng goto out_unlock; 184310183a69SYan, Zheng } 184410183a69SYan, Zheng 184510183a69SYan, Zheng rd_req->r_flags = CEPH_OSD_FLAG_READ; 184610183a69SYan, Zheng osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); 184710183a69SYan, Zheng rd_req->r_base_oloc.pool = pool; 1848779fe0fbSYan, Zheng if (pool_ns) 1849779fe0fbSYan, Zheng rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns); 1850d30291b9SIlya Dryomov ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ci->i_vino.ino); 185110183a69SYan, Zheng 185213d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(rd_req, GFP_NOFS); 185313d1ad16SIlya Dryomov if (err) 185413d1ad16SIlya Dryomov goto out_unlock; 185510183a69SYan, Zheng 185634b759b4SIlya Dryomov wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, 185710183a69SYan, Zheng 1, false, GFP_NOFS); 185810183a69SYan, Zheng if (!wr_req) { 185910183a69SYan, Zheng err = -ENOMEM; 186010183a69SYan, Zheng goto out_unlock; 186110183a69SYan, Zheng } 186210183a69SYan, Zheng 186354ea0046SIlya Dryomov wr_req->r_flags = CEPH_OSD_FLAG_WRITE; 186410183a69SYan, Zheng osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); 186563244fa1SIlya Dryomov ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc); 1866d30291b9SIlya Dryomov ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid); 186710183a69SYan, Zheng 186813d1ad16SIlya Dryomov err = ceph_osdc_alloc_messages(wr_req, GFP_NOFS); 186913d1ad16SIlya Dryomov if (err) 187013d1ad16SIlya Dryomov goto out_unlock; 187110183a69SYan, Zheng 187210183a69SYan, Zheng /* one page should be large enough for STAT data */ 187310183a69SYan, Zheng pages = ceph_alloc_page_vector(1, GFP_KERNEL); 187410183a69SYan, Zheng if (IS_ERR(pages)) { 187510183a69SYan, Zheng err = PTR_ERR(pages); 187610183a69SYan, Zheng goto out_unlock; 187710183a69SYan, Zheng } 187810183a69SYan, Zheng 187910183a69SYan, Zheng osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, 188010183a69SYan, Zheng 0, false, true); 188110183a69SYan, Zheng err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false); 188210183a69SYan, Zheng 1883fac02ddfSArnd Bergmann wr_req->r_mtime = ci->vfs_inode.i_mtime; 188410183a69SYan, Zheng err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false); 188510183a69SYan, Zheng 188610183a69SYan, Zheng if (!err) 188710183a69SYan, Zheng err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); 188810183a69SYan, Zheng if (!err2) 188910183a69SYan, Zheng err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); 189010183a69SYan, Zheng 189110183a69SYan, Zheng if (err >= 0 || err == -ENOENT) 189210183a69SYan, Zheng have |= POOL_READ; 1893131d7eb4SYan, Zheng else if (err != -EPERM) { 18940b98acd6SIlya Dryomov if (err == -EBLOCKLISTED) 18950b98acd6SIlya Dryomov fsc->blocklisted = true; 189610183a69SYan, Zheng goto out_unlock; 1897131d7eb4SYan, Zheng } 189810183a69SYan, Zheng 189910183a69SYan, Zheng if (err2 == 0 || err2 == -EEXIST) 190010183a69SYan, Zheng have |= POOL_WRITE; 190110183a69SYan, Zheng else if (err2 != -EPERM) { 19020b98acd6SIlya Dryomov if (err2 == -EBLOCKLISTED) 19030b98acd6SIlya Dryomov fsc->blocklisted = true; 190410183a69SYan, Zheng err = err2; 190510183a69SYan, Zheng goto out_unlock; 190610183a69SYan, Zheng } 190710183a69SYan, Zheng 1908779fe0fbSYan, Zheng pool_ns_len = pool_ns ? pool_ns->len : 0; 1909779fe0fbSYan, Zheng perm = kmalloc(sizeof(*perm) + pool_ns_len + 1, GFP_NOFS); 191010183a69SYan, Zheng if (!perm) { 191110183a69SYan, Zheng err = -ENOMEM; 191210183a69SYan, Zheng goto out_unlock; 191310183a69SYan, Zheng } 191410183a69SYan, Zheng 191510183a69SYan, Zheng perm->pool = pool; 191610183a69SYan, Zheng perm->perm = have; 1917779fe0fbSYan, Zheng perm->pool_ns_len = pool_ns_len; 1918779fe0fbSYan, Zheng if (pool_ns_len > 0) 1919779fe0fbSYan, Zheng memcpy(perm->pool_ns, pool_ns->str, pool_ns_len); 1920779fe0fbSYan, Zheng perm->pool_ns[pool_ns_len] = 0; 1921779fe0fbSYan, Zheng 192210183a69SYan, Zheng rb_link_node(&perm->node, parent, p); 192310183a69SYan, Zheng rb_insert_color(&perm->node, &mdsc->pool_perm_tree); 192410183a69SYan, Zheng err = 0; 192510183a69SYan, Zheng out_unlock: 192610183a69SYan, Zheng up_write(&mdsc->pool_perm_rwsem); 192710183a69SYan, Zheng 192810183a69SYan, Zheng ceph_osdc_put_request(rd_req); 192910183a69SYan, Zheng ceph_osdc_put_request(wr_req); 193010183a69SYan, Zheng out: 193110183a69SYan, Zheng if (!err) 193210183a69SYan, Zheng err = have; 1933779fe0fbSYan, Zheng if (pool_ns) 1934779fe0fbSYan, Zheng dout("__ceph_pool_perm_get pool %lld ns %.*s result = %d\n", 1935779fe0fbSYan, Zheng pool, (int)pool_ns->len, pool_ns->str, err); 1936779fe0fbSYan, Zheng else 19377627151eSYan, Zheng dout("__ceph_pool_perm_get pool %lld result = %d\n", pool, err); 193810183a69SYan, Zheng return err; 193910183a69SYan, Zheng } 194010183a69SYan, Zheng 19415e3ded1bSYan, Zheng int ceph_pool_perm_check(struct inode *inode, int need) 194210183a69SYan, Zheng { 19435e3ded1bSYan, Zheng struct ceph_inode_info *ci = ceph_inode(inode); 1944779fe0fbSYan, Zheng struct ceph_string *pool_ns; 19455e3ded1bSYan, Zheng s64 pool; 194610183a69SYan, Zheng int ret, flags; 194710183a69SYan, Zheng 194880e80fbbSYan, Zheng if (ci->i_vino.snap != CEPH_NOSNAP) { 194980e80fbbSYan, Zheng /* 195080e80fbbSYan, Zheng * Pool permission check needs to write to the first object. 195180e80fbbSYan, Zheng * But for snapshot, head of the first object may have alread 195280e80fbbSYan, Zheng * been deleted. Skip check to avoid creating orphan object. 195380e80fbbSYan, Zheng */ 195480e80fbbSYan, Zheng return 0; 195580e80fbbSYan, Zheng } 195680e80fbbSYan, Zheng 19575e3ded1bSYan, Zheng if (ceph_test_mount_opt(ceph_inode_to_client(inode), 195810183a69SYan, Zheng NOPOOLPERM)) 195910183a69SYan, Zheng return 0; 196010183a69SYan, Zheng 196110183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 196210183a69SYan, Zheng flags = ci->i_ceph_flags; 19637627151eSYan, Zheng pool = ci->i_layout.pool_id; 196410183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 196510183a69SYan, Zheng check: 196610183a69SYan, Zheng if (flags & CEPH_I_POOL_PERM) { 196710183a69SYan, Zheng if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { 19687627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no read perm\n", 196910183a69SYan, Zheng pool); 197010183a69SYan, Zheng return -EPERM; 197110183a69SYan, Zheng } 197210183a69SYan, Zheng if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { 19737627151eSYan, Zheng dout("ceph_pool_perm_check pool %lld no write perm\n", 197410183a69SYan, Zheng pool); 197510183a69SYan, Zheng return -EPERM; 197610183a69SYan, Zheng } 197710183a69SYan, Zheng return 0; 197810183a69SYan, Zheng } 197910183a69SYan, Zheng 1980779fe0fbSYan, Zheng pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); 1981779fe0fbSYan, Zheng ret = __ceph_pool_perm_get(ci, pool, pool_ns); 1982779fe0fbSYan, Zheng ceph_put_string(pool_ns); 198310183a69SYan, Zheng if (ret < 0) 198410183a69SYan, Zheng return ret; 198510183a69SYan, Zheng 198610183a69SYan, Zheng flags = CEPH_I_POOL_PERM; 198710183a69SYan, Zheng if (ret & POOL_READ) 198810183a69SYan, Zheng flags |= CEPH_I_POOL_RD; 198910183a69SYan, Zheng if (ret & POOL_WRITE) 199010183a69SYan, Zheng flags |= CEPH_I_POOL_WR; 199110183a69SYan, Zheng 199210183a69SYan, Zheng spin_lock(&ci->i_ceph_lock); 1993779fe0fbSYan, Zheng if (pool == ci->i_layout.pool_id && 1994779fe0fbSYan, Zheng pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { 1995779fe0fbSYan, Zheng ci->i_ceph_flags |= flags; 199610183a69SYan, Zheng } else { 19977627151eSYan, Zheng pool = ci->i_layout.pool_id; 199810183a69SYan, Zheng flags = ci->i_ceph_flags; 199910183a69SYan, Zheng } 200010183a69SYan, Zheng spin_unlock(&ci->i_ceph_lock); 200110183a69SYan, Zheng goto check; 200210183a69SYan, Zheng } 200310183a69SYan, Zheng 200410183a69SYan, Zheng void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) 200510183a69SYan, Zheng { 200610183a69SYan, Zheng struct ceph_pool_perm *perm; 200710183a69SYan, Zheng struct rb_node *n; 200810183a69SYan, Zheng 200910183a69SYan, Zheng while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { 201010183a69SYan, Zheng n = rb_first(&mdsc->pool_perm_tree); 201110183a69SYan, Zheng perm = rb_entry(n, struct ceph_pool_perm, node); 201210183a69SYan, Zheng rb_erase(n, &mdsc->pool_perm_tree); 201310183a69SYan, Zheng kfree(perm); 201410183a69SYan, Zheng } 201510183a69SYan, Zheng } 2016