1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/ceph/ceph_debug.h> 3 4 #include <linux/module.h> 5 #include <linux/sched.h> 6 #include <linux/slab.h> 7 #include <linux/file.h> 8 #include <linux/namei.h> 9 #include <linux/writeback.h> 10 11 #include <linux/ceph/libceph.h> 12 13 /* 14 * build a vector of user pages 15 */ 16 struct page **ceph_get_direct_page_vector(const void __user *data, 17 int num_pages, bool write_page) 18 { 19 struct page **pages; 20 int got = 0; 21 int rc = 0; 22 23 pages = kmalloc_array(num_pages, sizeof(*pages), GFP_NOFS); 24 if (!pages) 25 return ERR_PTR(-ENOMEM); 26 27 while (got < num_pages) { 28 rc = get_user_pages_fast( 29 (unsigned long)data + ((unsigned long)got * PAGE_SIZE), 30 num_pages - got, write_page ? FOLL_WRITE : 0, pages + got); 31 if (rc < 0) 32 break; 33 BUG_ON(rc == 0); 34 got += rc; 35 } 36 if (rc < 0) 37 goto fail; 38 return pages; 39 40 fail: 41 ceph_put_page_vector(pages, got, false); 42 return ERR_PTR(rc); 43 } 44 EXPORT_SYMBOL(ceph_get_direct_page_vector); 45 46 void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) 47 { 48 int i; 49 50 for (i = 0; i < num_pages; i++) { 51 if (dirty) 52 set_page_dirty_lock(pages[i]); 53 put_page(pages[i]); 54 } 55 kvfree(pages); 56 } 57 EXPORT_SYMBOL(ceph_put_page_vector); 58 59 void ceph_release_page_vector(struct page **pages, int num_pages) 60 { 61 int i; 62 63 for (i = 0; i < num_pages; i++) 64 __free_pages(pages[i], 0); 65 kfree(pages); 66 } 67 EXPORT_SYMBOL(ceph_release_page_vector); 68 69 /* 70 * allocate a vector new pages 71 */ 72 struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) 73 { 74 struct page **pages; 75 int i; 76 77 pages = kmalloc_array(num_pages, sizeof(*pages), flags); 78 if (!pages) 79 return ERR_PTR(-ENOMEM); 80 for (i = 0; i < num_pages; i++) { 81 pages[i] = __page_cache_alloc(flags); 82 if (pages[i] == NULL) { 83 ceph_release_page_vector(pages, i); 84 return ERR_PTR(-ENOMEM); 85 } 86 } 87 return pages; 88 } 89 EXPORT_SYMBOL(ceph_alloc_page_vector); 90 91 /* 92 * copy user data into a page vector 93 */ 94 int ceph_copy_user_to_page_vector(struct page **pages, 95 const void __user *data, 96 loff_t off, size_t len) 97 { 98 int i = 0; 99 int po = off & ~PAGE_MASK; 100 int left = len; 101 int l, bad; 102 103 while (left > 0) { 104 l = min_t(int, PAGE_SIZE-po, left); 105 bad = copy_from_user(page_address(pages[i]) + po, data, l); 106 if (bad == l) 107 return -EFAULT; 108 data += l - bad; 109 left -= l - bad; 110 po += l - bad; 111 if (po == PAGE_SIZE) { 112 po = 0; 113 i++; 114 } 115 } 116 return len; 117 } 118 EXPORT_SYMBOL(ceph_copy_user_to_page_vector); 119 120 void ceph_copy_to_page_vector(struct page **pages, 121 const void *data, 122 loff_t off, size_t len) 123 { 124 int i = 0; 125 size_t po = off & ~PAGE_MASK; 126 size_t left = len; 127 128 while (left > 0) { 129 size_t l = min_t(size_t, PAGE_SIZE-po, left); 130 131 memcpy(page_address(pages[i]) + po, data, l); 132 data += l; 133 left -= l; 134 po += l; 135 if (po == PAGE_SIZE) { 136 po = 0; 137 i++; 138 } 139 } 140 } 141 EXPORT_SYMBOL(ceph_copy_to_page_vector); 142 143 void ceph_copy_from_page_vector(struct page **pages, 144 void *data, 145 loff_t off, size_t len) 146 { 147 int i = 0; 148 size_t po = off & ~PAGE_MASK; 149 size_t left = len; 150 151 while (left > 0) { 152 size_t l = min_t(size_t, PAGE_SIZE-po, left); 153 154 memcpy(data, page_address(pages[i]) + po, l); 155 data += l; 156 left -= l; 157 po += l; 158 if (po == PAGE_SIZE) { 159 po = 0; 160 i++; 161 } 162 } 163 } 164 EXPORT_SYMBOL(ceph_copy_from_page_vector); 165 166 /* 167 * Zero an extent within a page vector. Offset is relative to the 168 * start of the first page. 169 */ 170 void ceph_zero_page_vector_range(int off, int len, struct page **pages) 171 { 172 int i = off >> PAGE_SHIFT; 173 174 off &= ~PAGE_MASK; 175 176 dout("zero_page_vector_page %u~%u\n", off, len); 177 178 /* leading partial page? */ 179 if (off) { 180 int end = min((int)PAGE_SIZE, off + len); 181 dout("zeroing %d %p head from %d\n", i, pages[i], 182 (int)off); 183 zero_user_segment(pages[i], off, end); 184 len -= (end - off); 185 i++; 186 } 187 while (len >= PAGE_SIZE) { 188 dout("zeroing %d %p len=%d\n", i, pages[i], len); 189 zero_user_segment(pages[i], 0, PAGE_SIZE); 190 len -= PAGE_SIZE; 191 i++; 192 } 193 /* trailing partial page? */ 194 if (len) { 195 dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); 196 zero_user_segment(pages[i], 0, len); 197 } 198 } 199 EXPORT_SYMBOL(ceph_zero_page_vector_range); 200