1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 23d14c5d2SYehuda Sadeh #include <linux/ceph/ceph_debug.h> 33d14c5d2SYehuda Sadeh 43d14c5d2SYehuda Sadeh #include <linux/module.h> 53d14c5d2SYehuda Sadeh #include <linux/sched.h> 63d14c5d2SYehuda Sadeh #include <linux/slab.h> 73d14c5d2SYehuda Sadeh #include <linux/file.h> 83d14c5d2SYehuda Sadeh #include <linux/namei.h> 93d14c5d2SYehuda Sadeh #include <linux/writeback.h> 103d14c5d2SYehuda Sadeh 113d14c5d2SYehuda Sadeh #include <linux/ceph/libceph.h> 123d14c5d2SYehuda Sadeh 133d14c5d2SYehuda Sadeh /* 143d14c5d2SYehuda Sadeh * build a vector of user pages 153d14c5d2SYehuda Sadeh */ 16b324814eSAlex Elder struct page **ceph_get_direct_page_vector(const void __user *data, 17b6aa5901SHenry C Chang int num_pages, bool write_page) 183d14c5d2SYehuda Sadeh { 193d14c5d2SYehuda Sadeh struct page **pages; 2038815b78SSage Weil int got = 0; 2138815b78SSage Weil int rc = 0; 223d14c5d2SYehuda Sadeh 233d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS); 243d14c5d2SYehuda Sadeh if (!pages) 253d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 263d14c5d2SYehuda Sadeh 2738815b78SSage Weil while (got < num_pages) { 28d4edcf0dSDave Hansen rc = get_user_pages_unlocked( 2938815b78SSage Weil (unsigned long)data + ((unsigned long)got * PAGE_SIZE), 30c164154fSLorenzo Stoakes num_pages - got, pages + got, write_page ? FOLL_WRITE : 0); 3138815b78SSage Weil if (rc < 0) 3238815b78SSage Weil break; 3338815b78SSage Weil BUG_ON(rc == 0); 3438815b78SSage Weil got += rc; 3538815b78SSage Weil } 3638815b78SSage Weil if (rc < 0) 373d14c5d2SYehuda Sadeh goto fail; 383d14c5d2SYehuda Sadeh return pages; 393d14c5d2SYehuda Sadeh 403d14c5d2SYehuda Sadeh fail: 4138815b78SSage Weil ceph_put_page_vector(pages, got, false); 423d14c5d2SYehuda Sadeh return ERR_PTR(rc); 433d14c5d2SYehuda Sadeh } 443d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_get_direct_page_vector); 453d14c5d2SYehuda Sadeh 46b6aa5901SHenry C Chang void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) 473d14c5d2SYehuda Sadeh { 483d14c5d2SYehuda Sadeh int i; 493d14c5d2SYehuda Sadeh 50b6aa5901SHenry C Chang for (i = 0; i < num_pages; i++) { 51b6aa5901SHenry C Chang if (dirty) 52b6aa5901SHenry C Chang set_page_dirty_lock(pages[i]); 533d14c5d2SYehuda Sadeh put_page(pages[i]); 54b6aa5901SHenry C Chang } 55b01da6a0SIlya Dryomov kvfree(pages); 563d14c5d2SYehuda Sadeh } 573d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_put_page_vector); 583d14c5d2SYehuda Sadeh 593d14c5d2SYehuda Sadeh void ceph_release_page_vector(struct page **pages, int num_pages) 603d14c5d2SYehuda Sadeh { 613d14c5d2SYehuda Sadeh int i; 623d14c5d2SYehuda Sadeh 633d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) 643d14c5d2SYehuda Sadeh __free_pages(pages[i], 0); 653d14c5d2SYehuda Sadeh kfree(pages); 663d14c5d2SYehuda Sadeh } 673d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_release_page_vector); 683d14c5d2SYehuda Sadeh 693d14c5d2SYehuda Sadeh /* 703d14c5d2SYehuda Sadeh * allocate a vector new pages 713d14c5d2SYehuda Sadeh */ 723d14c5d2SYehuda Sadeh struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) 733d14c5d2SYehuda Sadeh { 743d14c5d2SYehuda Sadeh struct page **pages; 753d14c5d2SYehuda Sadeh int i; 763d14c5d2SYehuda Sadeh 773d14c5d2SYehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, flags); 783d14c5d2SYehuda Sadeh if (!pages) 793d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 803d14c5d2SYehuda Sadeh for (i = 0; i < num_pages; i++) { 813d14c5d2SYehuda Sadeh pages[i] = __page_cache_alloc(flags); 823d14c5d2SYehuda Sadeh if (pages[i] == NULL) { 833d14c5d2SYehuda Sadeh ceph_release_page_vector(pages, i); 843d14c5d2SYehuda Sadeh return ERR_PTR(-ENOMEM); 853d14c5d2SYehuda Sadeh } 863d14c5d2SYehuda Sadeh } 873d14c5d2SYehuda Sadeh return pages; 883d14c5d2SYehuda Sadeh } 893d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_alloc_page_vector); 903d14c5d2SYehuda Sadeh 913d14c5d2SYehuda Sadeh /* 923d14c5d2SYehuda Sadeh * copy user data into a page vector 933d14c5d2SYehuda Sadeh */ 943d14c5d2SYehuda Sadeh int ceph_copy_user_to_page_vector(struct page **pages, 95b324814eSAlex Elder const void __user *data, 963d14c5d2SYehuda Sadeh loff_t off, size_t len) 973d14c5d2SYehuda Sadeh { 983d14c5d2SYehuda Sadeh int i = 0; 9909cbfeafSKirill A. Shutemov int po = off & ~PAGE_MASK; 1003d14c5d2SYehuda Sadeh int left = len; 1013d14c5d2SYehuda Sadeh int l, bad; 1023d14c5d2SYehuda Sadeh 1033d14c5d2SYehuda Sadeh while (left > 0) { 10409cbfeafSKirill A. Shutemov l = min_t(int, PAGE_SIZE-po, left); 1053d14c5d2SYehuda Sadeh bad = copy_from_user(page_address(pages[i]) + po, data, l); 1063d14c5d2SYehuda Sadeh if (bad == l) 1073d14c5d2SYehuda Sadeh return -EFAULT; 1083d14c5d2SYehuda Sadeh data += l - bad; 1093d14c5d2SYehuda Sadeh left -= l - bad; 1103d14c5d2SYehuda Sadeh po += l - bad; 11109cbfeafSKirill A. Shutemov if (po == PAGE_SIZE) { 1123d14c5d2SYehuda Sadeh po = 0; 1133d14c5d2SYehuda Sadeh i++; 1143d14c5d2SYehuda Sadeh } 1153d14c5d2SYehuda Sadeh } 1163d14c5d2SYehuda Sadeh return len; 1173d14c5d2SYehuda Sadeh } 1183d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_user_to_page_vector); 1193d14c5d2SYehuda Sadeh 120903bb32eSAlex Elder void ceph_copy_to_page_vector(struct page **pages, 121b324814eSAlex Elder const void *data, 1223d14c5d2SYehuda Sadeh loff_t off, size_t len) 1233d14c5d2SYehuda Sadeh { 1243d14c5d2SYehuda Sadeh int i = 0; 12509cbfeafSKirill A. Shutemov size_t po = off & ~PAGE_MASK; 1263d14c5d2SYehuda Sadeh size_t left = len; 1273d14c5d2SYehuda Sadeh 1283d14c5d2SYehuda Sadeh while (left > 0) { 12909cbfeafSKirill A. Shutemov size_t l = min_t(size_t, PAGE_SIZE-po, left); 130903bb32eSAlex Elder 1313d14c5d2SYehuda Sadeh memcpy(page_address(pages[i]) + po, data, l); 1323d14c5d2SYehuda Sadeh data += l; 1333d14c5d2SYehuda Sadeh left -= l; 1343d14c5d2SYehuda Sadeh po += l; 13509cbfeafSKirill A. Shutemov if (po == PAGE_SIZE) { 1363d14c5d2SYehuda Sadeh po = 0; 1373d14c5d2SYehuda Sadeh i++; 1383d14c5d2SYehuda Sadeh } 1393d14c5d2SYehuda Sadeh } 1403d14c5d2SYehuda Sadeh } 1413d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_to_page_vector); 1423d14c5d2SYehuda Sadeh 143903bb32eSAlex Elder void ceph_copy_from_page_vector(struct page **pages, 144b324814eSAlex Elder void *data, 1453d14c5d2SYehuda Sadeh loff_t off, size_t len) 1463d14c5d2SYehuda Sadeh { 1473d14c5d2SYehuda Sadeh int i = 0; 14809cbfeafSKirill A. Shutemov size_t po = off & ~PAGE_MASK; 1493d14c5d2SYehuda Sadeh size_t left = len; 1503d14c5d2SYehuda Sadeh 1513d14c5d2SYehuda Sadeh while (left > 0) { 15209cbfeafSKirill A. Shutemov size_t l = min_t(size_t, PAGE_SIZE-po, left); 153903bb32eSAlex Elder 1543d14c5d2SYehuda Sadeh memcpy(data, page_address(pages[i]) + po, l); 1553d14c5d2SYehuda Sadeh data += l; 1563d14c5d2SYehuda Sadeh left -= l; 1573d14c5d2SYehuda Sadeh po += l; 15809cbfeafSKirill A. Shutemov if (po == PAGE_SIZE) { 1593d14c5d2SYehuda Sadeh po = 0; 1603d14c5d2SYehuda Sadeh i++; 1613d14c5d2SYehuda Sadeh } 1623d14c5d2SYehuda Sadeh } 1633d14c5d2SYehuda Sadeh } 1643d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_copy_from_page_vector); 1653d14c5d2SYehuda Sadeh 1663d14c5d2SYehuda Sadeh /* 1673d14c5d2SYehuda Sadeh * Zero an extent within a page vector. Offset is relative to the 1683d14c5d2SYehuda Sadeh * start of the first page. 1693d14c5d2SYehuda Sadeh */ 1703d14c5d2SYehuda Sadeh void ceph_zero_page_vector_range(int off, int len, struct page **pages) 1713d14c5d2SYehuda Sadeh { 17209cbfeafSKirill A. Shutemov int i = off >> PAGE_SHIFT; 1733d14c5d2SYehuda Sadeh 17409cbfeafSKirill A. Shutemov off &= ~PAGE_MASK; 1753d14c5d2SYehuda Sadeh 1763d14c5d2SYehuda Sadeh dout("zero_page_vector_page %u~%u\n", off, len); 1773d14c5d2SYehuda Sadeh 1783d14c5d2SYehuda Sadeh /* leading partial page? */ 1793d14c5d2SYehuda Sadeh if (off) { 18009cbfeafSKirill A. Shutemov int end = min((int)PAGE_SIZE, off + len); 1813d14c5d2SYehuda Sadeh dout("zeroing %d %p head from %d\n", i, pages[i], 1823d14c5d2SYehuda Sadeh (int)off); 1833d14c5d2SYehuda Sadeh zero_user_segment(pages[i], off, end); 1843d14c5d2SYehuda Sadeh len -= (end - off); 1853d14c5d2SYehuda Sadeh i++; 1863d14c5d2SYehuda Sadeh } 18709cbfeafSKirill A. Shutemov while (len >= PAGE_SIZE) { 1883d14c5d2SYehuda Sadeh dout("zeroing %d %p len=%d\n", i, pages[i], len); 18909cbfeafSKirill A. Shutemov zero_user_segment(pages[i], 0, PAGE_SIZE); 19009cbfeafSKirill A. Shutemov len -= PAGE_SIZE; 1913d14c5d2SYehuda Sadeh i++; 1923d14c5d2SYehuda Sadeh } 1933d14c5d2SYehuda Sadeh /* trailing partial page? */ 1943d14c5d2SYehuda Sadeh if (len) { 1953d14c5d2SYehuda Sadeh dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); 1963d14c5d2SYehuda Sadeh zero_user_segment(pages[i], 0, len); 1973d14c5d2SYehuda Sadeh } 1983d14c5d2SYehuda Sadeh } 1993d14c5d2SYehuda Sadeh EXPORT_SYMBOL(ceph_zero_page_vector_range); 2003d14c5d2SYehuda Sadeh 201