13d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 23d14c5d2SYehuda Sadeh 33d14c5d2SYehuda Sadeh #include <linux/module.h> 43d14c5d2SYehuda Sadeh #include <linux/sched.h> 53d14c5d2SYehuda Sadeh #include <linux/slab.h> 63d14c5d2SYehuda Sadeh #include <linux/file.h> 73d14c5d2SYehuda Sadeh #include <linux/namei.h> 83d14c5d2SYehuda Sadeh #include <linux/writeback.h> 93d14c5d2SYehuda Sadeh 103d14c5d2SYehuda Sadeh #include <linux/ceph/libceph.h> 113d14c5d2SYehuda Sadeh 123d14c5d2SYehuda Sadeh /* 133d14c5d2SYehuda Sadeh * build a vector of user pages 143d14c5d2SYehuda Sadeh */ 153d14c5d2SYehuda Sadeh struct page **ceph_get_direct_page_vector(const char __user *data, 16b6aa5901SHenry C Chang int num_pages, bool write_page) 173d14c5d2SYehuda Sadeh { 183d14c5d2SYehuda Sadeh struct page **pages; 1938815b78SSage Weil int got = 0; 2038815b78SSage Weil int rc = 0; 213d14c5d2SYehuda Sadeh 223d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS); 233d14c5d2SYehuda Sadeh if (!pages) 243d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 253d14c5d2SYehuda Sadeh 263d14c5d2SYehuda Sadeh down_read(¤t->mm->mmap_sem); 2738815b78SSage Weil while (got < num_pages) { 2838815b78SSage Weil rc = get_user_pages(current, current->mm, 2938815b78SSage Weil (unsigned long)data + ((unsigned long)got * PAGE_SIZE), 3038815b78SSage Weil num_pages - got, write_page, 0, pages + got, NULL); 3138815b78SSage Weil if (rc < 0) 3238815b78SSage Weil break; 3338815b78SSage Weil BUG_ON(rc == 0); 3438815b78SSage Weil got += rc; 3538815b78SSage Weil } 363d14c5d2SYehuda Sadeh up_read(¤t->mm->mmap_sem); 3738815b78SSage Weil if (rc < 0) 383d14c5d2SYehuda Sadeh goto fail; 393d14c5d2SYehuda Sadeh return pages; 403d14c5d2SYehuda Sadeh 413d14c5d2SYehuda Sadeh fail: 4238815b78SSage Weil ceph_put_page_vector(pages, got, false); 433d14c5d2SYehuda Sadeh return ERR_PTR(rc); 443d14c5d2SYehuda Sadeh } 453d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_get_direct_page_vector); 463d14c5d2SYehuda Sadeh 47b6aa5901SHenry C Chang void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) 483d14c5d2SYehuda Sadeh { 493d14c5d2SYehuda Sadeh int i; 503d14c5d2SYehuda Sadeh 51b6aa5901SHenry C Chang for (i = 0; i < num_pages; i++) { 52b6aa5901SHenry C Chang if (dirty) 53b6aa5901SHenry C Chang set_page_dirty_lock(pages[i]); 543d14c5d2SYehuda Sadeh put_page(pages[i]); 55b6aa5901SHenry C Chang } 563d14c5d2SYehuda Sadeh kfree(pages); 573d14c5d2SYehuda Sadeh } 583d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_put_page_vector); 593d14c5d2SYehuda Sadeh 603d14c5d2SYehuda Sadeh void ceph_release_page_vector(struct page **pages, int num_pages) 613d14c5d2SYehuda Sadeh { 623d14c5d2SYehuda Sadeh int i; 633d14c5d2SYehuda Sadeh 643d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) 653d14c5d2SYehuda Sadeh __free_pages(pages[i], 0); 663d14c5d2SYehuda Sadeh kfree(pages); 673d14c5d2SYehuda Sadeh } 683d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_release_page_vector); 693d14c5d2SYehuda Sadeh 703d14c5d2SYehuda Sadeh /* 713d14c5d2SYehuda Sadeh * allocate a vector new pages 723d14c5d2SYehuda Sadeh */ 733d14c5d2SYehuda Sadeh struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) 743d14c5d2SYehuda Sadeh { 753d14c5d2SYehuda Sadeh struct page **pages; 763d14c5d2SYehuda Sadeh int i; 773d14c5d2SYehuda Sadeh 783d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, flags); 793d14c5d2SYehuda Sadeh if (!pages) 803d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 813d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) { 823d14c5d2SYehuda Sadeh pages[i] = __page_cache_alloc(flags); 833d14c5d2SYehuda Sadeh if (pages[i] == NULL) { 843d14c5d2SYehuda Sadeh ceph_release_page_vector(pages, i); 853d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 863d14c5d2SYehuda Sadeh } 873d14c5d2SYehuda Sadeh } 883d14c5d2SYehuda Sadeh return pages; 893d14c5d2SYehuda Sadeh } 903d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_alloc_page_vector); 913d14c5d2SYehuda Sadeh 923d14c5d2SYehuda Sadeh /* 933d14c5d2SYehuda Sadeh * copy user data into a page vector 943d14c5d2SYehuda Sadeh */ 953d14c5d2SYehuda Sadeh int ceph_copy_user_to_page_vector(struct page **pages, 963d14c5d2SYehuda Sadeh const char __user *data, 973d14c5d2SYehuda Sadeh loff_t off, size_t len) 983d14c5d2SYehuda Sadeh { 993d14c5d2SYehuda Sadeh int i = 0; 1003d14c5d2SYehuda Sadeh int po = off & ~PAGE_CACHE_MASK; 1013d14c5d2SYehuda Sadeh int left = len; 1023d14c5d2SYehuda Sadeh int l, bad; 1033d14c5d2SYehuda Sadeh 1043d14c5d2SYehuda Sadeh while (left > 0) { 1053d14c5d2SYehuda Sadeh l = min_t(int, PAGE_CACHE_SIZE-po, left); 1063d14c5d2SYehuda Sadeh bad = copy_from_user(page_address(pages[i]) + po, data, l); 1073d14c5d2SYehuda Sadeh if (bad == l) 1083d14c5d2SYehuda Sadeh return -EFAULT; 1093d14c5d2SYehuda Sadeh data += l - bad; 1103d14c5d2SYehuda Sadeh left -= l - bad; 1113d14c5d2SYehuda Sadeh po += l - bad; 1123d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1133d14c5d2SYehuda Sadeh po = 0; 1143d14c5d2SYehuda Sadeh i++; 1153d14c5d2SYehuda Sadeh } 1163d14c5d2SYehuda Sadeh } 1173d14c5d2SYehuda Sadeh return len; 1183d14c5d2SYehuda Sadeh } 1193d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_user_to_page_vector); 1203d14c5d2SYehuda Sadeh 1213d14c5d2SYehuda Sadeh int ceph_copy_to_page_vector(struct page **pages, 1223d14c5d2SYehuda Sadeh const char *data, 1233d14c5d2SYehuda Sadeh loff_t off, size_t len) 1243d14c5d2SYehuda Sadeh { 1253d14c5d2SYehuda Sadeh int i = 0; 1263d14c5d2SYehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1273d14c5d2SYehuda Sadeh size_t left = len; 1283d14c5d2SYehuda Sadeh size_t l; 1293d14c5d2SYehuda Sadeh 1303d14c5d2SYehuda Sadeh while (left > 0) { 1313d14c5d2SYehuda Sadeh l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 1323d14c5d2SYehuda Sadeh memcpy(page_address(pages[i]) + po, data, l); 1333d14c5d2SYehuda Sadeh data += l; 1343d14c5d2SYehuda Sadeh left -= l; 1353d14c5d2SYehuda Sadeh po += l; 1363d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1373d14c5d2SYehuda Sadeh po = 0; 1383d14c5d2SYehuda Sadeh i++; 1393d14c5d2SYehuda Sadeh } 1403d14c5d2SYehuda Sadeh } 1413d14c5d2SYehuda Sadeh return len; 1423d14c5d2SYehuda Sadeh } 1433d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_to_page_vector); 1443d14c5d2SYehuda Sadeh 1453d14c5d2SYehuda Sadeh int ceph_copy_from_page_vector(struct page **pages, 1463d14c5d2SYehuda Sadeh char *data, 1473d14c5d2SYehuda Sadeh loff_t off, size_t len) 1483d14c5d2SYehuda Sadeh { 1493d14c5d2SYehuda Sadeh int i = 0; 1503d14c5d2SYehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1513d14c5d2SYehuda Sadeh size_t left = len; 1523d14c5d2SYehuda Sadeh size_t l; 1533d14c5d2SYehuda Sadeh 1543d14c5d2SYehuda Sadeh while (left > 0) { 1553d14c5d2SYehuda Sadeh l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 1563d14c5d2SYehuda Sadeh memcpy(data, page_address(pages[i]) + po, l); 1573d14c5d2SYehuda Sadeh data += l; 1583d14c5d2SYehuda Sadeh left -= l; 1593d14c5d2SYehuda Sadeh po += l; 1603d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1613d14c5d2SYehuda Sadeh po = 0; 1623d14c5d2SYehuda Sadeh i++; 1633d14c5d2SYehuda Sadeh } 1643d14c5d2SYehuda Sadeh } 1653d14c5d2SYehuda Sadeh return len; 1663d14c5d2SYehuda Sadeh } 1673d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_from_page_vector); 1683d14c5d2SYehuda Sadeh 1693d14c5d2SYehuda Sadeh /* 1703d14c5d2SYehuda Sadeh * copy user data from a page vector into a user pointer 1713d14c5d2SYehuda Sadeh */ 1723d14c5d2SYehuda Sadeh int ceph_copy_page_vector_to_user(struct page **pages, 1733d14c5d2SYehuda Sadeh char __user *data, 1743d14c5d2SYehuda Sadeh loff_t off, size_t len) 1753d14c5d2SYehuda Sadeh { 1763d14c5d2SYehuda Sadeh int i = 0; 1773d14c5d2SYehuda Sadeh int po = off & ~PAGE_CACHE_MASK; 1783d14c5d2SYehuda Sadeh int left = len; 1793d14c5d2SYehuda Sadeh int l, bad; 1803d14c5d2SYehuda Sadeh 1813d14c5d2SYehuda Sadeh while (left > 0) { 1823d14c5d2SYehuda Sadeh l = min_t(int, left, PAGE_CACHE_SIZE-po); 1833d14c5d2SYehuda Sadeh bad = copy_to_user(data, page_address(pages[i]) + po, l); 1843d14c5d2SYehuda Sadeh if (bad == l) 1853d14c5d2SYehuda Sadeh return -EFAULT; 1863d14c5d2SYehuda Sadeh data += l - bad; 1873d14c5d2SYehuda Sadeh left -= l - bad; 1883d14c5d2SYehuda Sadeh if (po) { 1893d14c5d2SYehuda Sadeh po += l - bad; 1903d14c5d2SYehuda Sadeh if (po == PAGE_CACHE_SIZE) 1913d14c5d2SYehuda Sadeh po = 0; 1923d14c5d2SYehuda Sadeh } 1933d14c5d2SYehuda Sadeh i++; 1943d14c5d2SYehuda Sadeh } 1953d14c5d2SYehuda Sadeh return len; 1963d14c5d2SYehuda Sadeh } 1973d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_page_vector_to_user); 1983d14c5d2SYehuda Sadeh 1993d14c5d2SYehuda Sadeh /* 2003d14c5d2SYehuda Sadeh * Zero an extent within a page vector. Offset is relative to the 2013d14c5d2SYehuda Sadeh * start of the first page. 2023d14c5d2SYehuda Sadeh */ 2033d14c5d2SYehuda Sadeh void ceph_zero_page_vector_range(int off, int len, struct page **pages) 2043d14c5d2SYehuda Sadeh { 2053d14c5d2SYehuda Sadeh int i = off >> PAGE_CACHE_SHIFT; 2063d14c5d2SYehuda Sadeh 2073d14c5d2SYehuda Sadeh off &= ~PAGE_CACHE_MASK; 2083d14c5d2SYehuda Sadeh 2093d14c5d2SYehuda Sadeh dout("zero_page_vector_page %u~%u\n", off, len); 2103d14c5d2SYehuda Sadeh 2113d14c5d2SYehuda Sadeh /* leading partial page? */ 2123d14c5d2SYehuda Sadeh if (off) { 2133d14c5d2SYehuda Sadeh int end = min((int)PAGE_CACHE_SIZE, off + len); 2143d14c5d2SYehuda Sadeh dout("zeroing %d %p head from %d\n", i, pages[i], 2153d14c5d2SYehuda Sadeh (int)off); 2163d14c5d2SYehuda Sadeh zero_user_segment(pages[i], off, end); 2173d14c5d2SYehuda Sadeh len -= (end - off); 2183d14c5d2SYehuda Sadeh i++; 2193d14c5d2SYehuda Sadeh } 2203d14c5d2SYehuda Sadeh while (len >= PAGE_CACHE_SIZE) { 2213d14c5d2SYehuda Sadeh dout("zeroing %d %p len=%d\n", i, pages[i], len); 2223d14c5d2SYehuda Sadeh zero_user_segment(pages[i], 0, PAGE_CACHE_SIZE); 2233d14c5d2SYehuda Sadeh len -= PAGE_CACHE_SIZE; 2243d14c5d2SYehuda Sadeh i++; 2253d14c5d2SYehuda Sadeh } 2263d14c5d2SYehuda Sadeh /* trailing partial page? */ 2273d14c5d2SYehuda Sadeh if (len) { 2283d14c5d2SYehuda Sadeh dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); 2293d14c5d2SYehuda Sadeh zero_user_segment(pages[i], 0, len); 2303d14c5d2SYehuda Sadeh } 2313d14c5d2SYehuda Sadeh } 2323d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_zero_page_vector_range); 2333d14c5d2SYehuda Sadeh 234