13d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 23d14c5d2SYehuda Sadeh 33d14c5d2SYehuda Sadeh #include <linux/module.h> 43d14c5d2SYehuda Sadeh #include <linux/sched.h> 53d14c5d2SYehuda Sadeh #include <linux/slab.h> 63d14c5d2SYehuda Sadeh #include <linux/file.h> 73d14c5d2SYehuda Sadeh #include <linux/namei.h> 83d14c5d2SYehuda Sadeh #include <linux/writeback.h> 93d14c5d2SYehuda Sadeh 103d14c5d2SYehuda Sadeh #include <linux/ceph/libceph.h> 113d14c5d2SYehuda Sadeh 123d14c5d2SYehuda Sadeh /* 133d14c5d2SYehuda Sadeh * build a vector of user pages 143d14c5d2SYehuda Sadeh */ 153d14c5d2SYehuda Sadeh struct page **ceph_get_direct_page_vector(const char __user *data, 163d14c5d2SYehuda Sadeh int num_pages, 173d14c5d2SYehuda Sadeh loff_t off, size_t len) 183d14c5d2SYehuda Sadeh { 193d14c5d2SYehuda Sadeh struct page **pages; 203d14c5d2SYehuda Sadeh int rc; 213d14c5d2SYehuda Sadeh 223d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS); 233d14c5d2SYehuda Sadeh if (!pages) 243d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 253d14c5d2SYehuda Sadeh 263d14c5d2SYehuda Sadeh down_read(¤t->mm->mmap_sem); 273d14c5d2SYehuda Sadeh rc = get_user_pages(current, current->mm, (unsigned long)data, 283d14c5d2SYehuda Sadeh num_pages, 0, 0, pages, NULL); 293d14c5d2SYehuda Sadeh up_read(¤t->mm->mmap_sem); 303d14c5d2SYehuda Sadeh if (rc < 0) 313d14c5d2SYehuda Sadeh goto fail; 323d14c5d2SYehuda Sadeh return pages; 333d14c5d2SYehuda Sadeh 343d14c5d2SYehuda Sadeh fail: 353d14c5d2SYehuda Sadeh kfree(pages); 363d14c5d2SYehuda Sadeh return ERR_PTR(rc); 373d14c5d2SYehuda Sadeh } 383d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_get_direct_page_vector); 393d14c5d2SYehuda Sadeh 403d14c5d2SYehuda Sadeh void ceph_put_page_vector(struct page **pages, int num_pages) 413d14c5d2SYehuda Sadeh { 423d14c5d2SYehuda Sadeh int i; 433d14c5d2SYehuda Sadeh 443d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) 453d14c5d2SYehuda Sadeh put_page(pages[i]); 463d14c5d2SYehuda Sadeh kfree(pages); 473d14c5d2SYehuda Sadeh } 483d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_put_page_vector); 493d14c5d2SYehuda Sadeh 503d14c5d2SYehuda Sadeh void ceph_release_page_vector(struct page **pages, int num_pages) 513d14c5d2SYehuda Sadeh { 523d14c5d2SYehuda Sadeh int i; 533d14c5d2SYehuda Sadeh 543d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) 553d14c5d2SYehuda Sadeh __free_pages(pages[i], 0); 563d14c5d2SYehuda Sadeh kfree(pages); 573d14c5d2SYehuda Sadeh } 583d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_release_page_vector); 593d14c5d2SYehuda Sadeh 603d14c5d2SYehuda Sadeh /* 613d14c5d2SYehuda Sadeh * allocate a vector new pages 623d14c5d2SYehuda Sadeh */ 633d14c5d2SYehuda Sadeh struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) 643d14c5d2SYehuda Sadeh { 653d14c5d2SYehuda Sadeh struct page **pages; 663d14c5d2SYehuda Sadeh int i; 673d14c5d2SYehuda Sadeh 683d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, flags); 693d14c5d2SYehuda Sadeh if (!pages) 703d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 713d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) { 723d14c5d2SYehuda Sadeh pages[i] = __page_cache_alloc(flags); 733d14c5d2SYehuda Sadeh if (pages[i] == NULL) { 743d14c5d2SYehuda Sadeh ceph_release_page_vector(pages, i); 753d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 763d14c5d2SYehuda Sadeh } 773d14c5d2SYehuda Sadeh } 783d14c5d2SYehuda Sadeh return pages; 793d14c5d2SYehuda Sadeh } 803d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_alloc_page_vector); 813d14c5d2SYehuda Sadeh 823d14c5d2SYehuda Sadeh /* 833d14c5d2SYehuda Sadeh * copy user data into a page vector 843d14c5d2SYehuda Sadeh */ 853d14c5d2SYehuda Sadeh int ceph_copy_user_to_page_vector(struct page **pages, 863d14c5d2SYehuda Sadeh const char __user *data, 873d14c5d2SYehuda Sadeh loff_t off, size_t len) 883d14c5d2SYehuda Sadeh { 893d14c5d2SYehuda Sadeh int i = 0; 903d14c5d2SYehuda Sadeh int po = off & ~PAGE_CACHE_MASK; 913d14c5d2SYehuda Sadeh int left = len; 923d14c5d2SYehuda Sadeh int l, bad; 933d14c5d2SYehuda Sadeh 943d14c5d2SYehuda Sadeh while (left > 0) { 953d14c5d2SYehuda Sadeh l = min_t(int, PAGE_CACHE_SIZE-po, left); 963d14c5d2SYehuda Sadeh bad = copy_from_user(page_address(pages[i]) + po, data, l); 973d14c5d2SYehuda Sadeh if (bad == l) 983d14c5d2SYehuda Sadeh return -EFAULT; 993d14c5d2SYehuda Sadeh data += l - bad; 1003d14c5d2SYehuda Sadeh left -= l - bad; 1013d14c5d2SYehuda Sadeh po += l - bad; 1023d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1033d14c5d2SYehuda Sadeh po = 0; 1043d14c5d2SYehuda Sadeh i++; 1053d14c5d2SYehuda Sadeh } 1063d14c5d2SYehuda Sadeh } 1073d14c5d2SYehuda Sadeh return len; 1083d14c5d2SYehuda Sadeh } 1093d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_user_to_page_vector); 1103d14c5d2SYehuda Sadeh 1113d14c5d2SYehuda Sadeh int ceph_copy_to_page_vector(struct page **pages, 1123d14c5d2SYehuda Sadeh const char *data, 1133d14c5d2SYehuda Sadeh loff_t off, size_t len) 1143d14c5d2SYehuda Sadeh { 1153d14c5d2SYehuda Sadeh int i = 0; 1163d14c5d2SYehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1173d14c5d2SYehuda Sadeh size_t left = len; 1183d14c5d2SYehuda Sadeh size_t l; 1193d14c5d2SYehuda Sadeh 1203d14c5d2SYehuda Sadeh while (left > 0) { 1213d14c5d2SYehuda Sadeh l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 1223d14c5d2SYehuda Sadeh memcpy(page_address(pages[i]) + po, data, l); 1233d14c5d2SYehuda Sadeh data += l; 1243d14c5d2SYehuda Sadeh left -= l; 1253d14c5d2SYehuda Sadeh po += l; 1263d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1273d14c5d2SYehuda Sadeh po = 0; 1283d14c5d2SYehuda Sadeh i++; 1293d14c5d2SYehuda Sadeh } 1303d14c5d2SYehuda Sadeh } 1313d14c5d2SYehuda Sadeh return len; 1323d14c5d2SYehuda Sadeh } 1333d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_to_page_vector); 1343d14c5d2SYehuda Sadeh 1353d14c5d2SYehuda Sadeh int ceph_copy_from_page_vector(struct page **pages, 1363d14c5d2SYehuda Sadeh char *data, 1373d14c5d2SYehuda Sadeh loff_t off, size_t len) 1383d14c5d2SYehuda Sadeh { 1393d14c5d2SYehuda Sadeh int i = 0; 1403d14c5d2SYehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1413d14c5d2SYehuda Sadeh size_t left = len; 1423d14c5d2SYehuda Sadeh size_t l; 1433d14c5d2SYehuda Sadeh 1443d14c5d2SYehuda Sadeh while (left > 0) { 1453d14c5d2SYehuda Sadeh l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 1463d14c5d2SYehuda Sadeh memcpy(data, page_address(pages[i]) + po, l); 1473d14c5d2SYehuda Sadeh data += l; 1483d14c5d2SYehuda Sadeh left -= l; 1493d14c5d2SYehuda Sadeh po += l; 1503d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1513d14c5d2SYehuda Sadeh po = 0; 1523d14c5d2SYehuda Sadeh i++; 1533d14c5d2SYehuda Sadeh } 1543d14c5d2SYehuda Sadeh } 1553d14c5d2SYehuda Sadeh return len; 1563d14c5d2SYehuda Sadeh } 1573d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_from_page_vector); 1583d14c5d2SYehuda Sadeh 1593d14c5d2SYehuda Sadeh /* 1603d14c5d2SYehuda Sadeh * copy user data from a page vector into a user pointer 1613d14c5d2SYehuda Sadeh */ 1623d14c5d2SYehuda Sadeh int ceph_copy_page_vector_to_user(struct page **pages, 1633d14c5d2SYehuda Sadeh char __user *data, 1643d14c5d2SYehuda Sadeh loff_t off, size_t len) 1653d14c5d2SYehuda Sadeh { 1663d14c5d2SYehuda Sadeh int i = 0; 1673d14c5d2SYehuda Sadeh int po = off & ~PAGE_CACHE_MASK; 1683d14c5d2SYehuda Sadeh int left = len; 1693d14c5d2SYehuda Sadeh int l, bad; 1703d14c5d2SYehuda Sadeh 1713d14c5d2SYehuda Sadeh while (left > 0) { 1723d14c5d2SYehuda Sadeh l = min_t(int, left, PAGE_CACHE_SIZE-po); 1733d14c5d2SYehuda Sadeh bad = copy_to_user(data, page_address(pages[i]) + po, l); 1743d14c5d2SYehuda Sadeh if (bad == l) 1753d14c5d2SYehuda Sadeh return -EFAULT; 1763d14c5d2SYehuda Sadeh data += l - bad; 1773d14c5d2SYehuda Sadeh left -= l - bad; 1783d14c5d2SYehuda Sadeh if (po) { 1793d14c5d2SYehuda Sadeh po += l - bad; 1803d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) 1813d14c5d2SYehuda Sadeh po = 0; 1823d14c5d2SYehuda Sadeh } 1833d14c5d2SYehuda Sadeh i++; 1843d14c5d2SYehuda Sadeh } 1853d14c5d2SYehuda Sadeh return len; 1863d14c5d2SYehuda Sadeh } 1873d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_page_vector_to_user); 1883d14c5d2SYehuda Sadeh 1893d14c5d2SYehuda Sadeh /* 1903d14c5d2SYehuda Sadeh * Zero an extent within a page vector. Offset is relative to the 1913d14c5d2SYehuda Sadeh * start of the first page. 1923d14c5d2SYehuda Sadeh */ 1933d14c5d2SYehuda Sadeh void ceph_zero_page_vector_range(int off, int len, struct page **pages) 1943d14c5d2SYehuda Sadeh { 1953d14c5d2SYehuda Sadeh int i = off >> PAGE_CACHE_SHIFT; 1963d14c5d2SYehuda Sadeh 1973d14c5d2SYehuda Sadeh off &= ~PAGE_CACHE_MASK; 1983d14c5d2SYehuda Sadeh 1993d14c5d2SYehuda Sadeh dout("zero_page_vector_page %u~%u\n", off, len); 2003d14c5d2SYehuda Sadeh 2013d14c5d2SYehuda Sadeh /* leading partial page? */ 2023d14c5d2SYehuda Sadeh if (off) { 2033d14c5d2SYehuda Sadeh int end = min((int)PAGE_CACHE_SIZE, off + len); 2043d14c5d2SYehuda Sadeh dout("zeroing %d %p head from %d\n", i, pages[i], 2053d14c5d2SYehuda Sadeh (int)off); 2063d14c5d2SYehuda Sadeh zero_user_segment(pages[i], off, end); 2073d14c5d2SYehuda Sadeh len -= (end - off); 2083d14c5d2SYehuda Sadeh i++; 2093d14c5d2SYehuda Sadeh } 2103d14c5d2SYehuda Sadeh while (len >= PAGE_CACHE_SIZE) { 2113d14c5d2SYehuda Sadeh dout("zeroing %d %p len=%d\n", i, pages[i], len); 2123d14c5d2SYehuda Sadeh zero_user_segment(pages[i], 0, PAGE_CACHE_SIZE); 2133d14c5d2SYehuda Sadeh len -= PAGE_CACHE_SIZE; 2143d14c5d2SYehuda Sadeh i++; 2153d14c5d2SYehuda Sadeh } 2163d14c5d2SYehuda Sadeh /* trailing partial page? */ 2173d14c5d2SYehuda Sadeh if (len) { 2183d14c5d2SYehuda Sadeh dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); 2193d14c5d2SYehuda Sadeh zero_user_segment(pages[i], 0, len); 2203d14c5d2SYehuda Sadeh } 2213d14c5d2SYehuda Sadeh } 2223d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_zero_page_vector_range); 2233d14c5d2SYehuda Sadeh 224