1 #include <linux/kernel.h> 2 #include <linux/mm.h> 3 #include <linux/slab.h> 4 #include <linux/uaccess.h> 5 #include <linux/ktime.h> 6 #include <linux/debugfs.h> 7 #include "gup_test.h" 8 9 static void put_back_pages(unsigned int cmd, struct page **pages, 10 unsigned long nr_pages, unsigned int gup_test_flags) 11 { 12 unsigned long i; 13 14 switch (cmd) { 15 case GUP_FAST_BENCHMARK: 16 case GUP_BASIC_TEST: 17 for (i = 0; i < nr_pages; i++) 18 put_page(pages[i]); 19 break; 20 21 case PIN_FAST_BENCHMARK: 22 case PIN_BASIC_TEST: 23 case PIN_LONGTERM_BENCHMARK: 24 unpin_user_pages(pages, nr_pages); 25 break; 26 case DUMP_USER_PAGES_TEST: 27 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) { 28 unpin_user_pages(pages, nr_pages); 29 } else { 30 for (i = 0; i < nr_pages; i++) 31 put_page(pages[i]); 32 33 } 34 break; 35 } 36 } 37 38 static void verify_dma_pinned(unsigned int cmd, struct page **pages, 39 unsigned long nr_pages) 40 { 41 unsigned long i; 42 struct page *page; 43 44 switch (cmd) { 45 case PIN_FAST_BENCHMARK: 46 case PIN_BASIC_TEST: 47 case PIN_LONGTERM_BENCHMARK: 48 for (i = 0; i < nr_pages; i++) { 49 page = pages[i]; 50 if (WARN(!page_maybe_dma_pinned(page), 51 "pages[%lu] is NOT dma-pinned\n", i)) { 52 53 dump_page(page, "gup_test failure"); 54 break; 55 } 56 } 57 break; 58 } 59 } 60 61 static void dump_pages_test(struct gup_test *gup, struct page **pages, 62 unsigned long nr_pages) 63 { 64 unsigned int index_to_dump; 65 unsigned int i; 66 67 /* 68 * Zero out any user-supplied page index that is out of range. Remember: 69 * .which_pages[] contains a 1-based set of page indices. 70 */ 71 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) { 72 if (gup->which_pages[i] > nr_pages) { 73 pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n", 74 i, gup->which_pages[i]); 75 gup->which_pages[i] = 0; 76 } 77 } 78 79 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) { 80 index_to_dump = gup->which_pages[i]; 81 82 if (index_to_dump) { 83 index_to_dump--; // Decode from 1-based, to 0-based 84 pr_info("---- page #%u, starting from user virt addr: 0x%llx\n", 85 index_to_dump, gup->addr); 86 dump_page(pages[index_to_dump], 87 "gup_test: dump_pages() test"); 88 } 89 } 90 } 91 92 static int __gup_test_ioctl(unsigned int cmd, 93 struct gup_test *gup) 94 { 95 ktime_t start_time, end_time; 96 unsigned long i, nr_pages, addr, next; 97 int nr; 98 struct page **pages; 99 int ret = 0; 100 bool needs_mmap_lock = 101 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; 102 103 if (gup->size > ULONG_MAX) 104 return -EINVAL; 105 106 nr_pages = gup->size / PAGE_SIZE; 107 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL); 108 if (!pages) 109 return -ENOMEM; 110 111 if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) { 112 ret = -EINTR; 113 goto free_pages; 114 } 115 116 i = 0; 117 nr = gup->nr_pages_per_call; 118 start_time = ktime_get(); 119 for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { 120 if (nr != gup->nr_pages_per_call) 121 break; 122 123 next = addr + nr * PAGE_SIZE; 124 if (next > gup->addr + gup->size) { 125 next = gup->addr + gup->size; 126 nr = (next - addr) / PAGE_SIZE; 127 } 128 129 /* Filter out most gup flags: only allow a tiny subset here: */ 130 gup->flags &= FOLL_WRITE; 131 132 switch (cmd) { 133 case GUP_FAST_BENCHMARK: 134 nr = get_user_pages_fast(addr, nr, gup->flags, 135 pages + i); 136 break; 137 case GUP_BASIC_TEST: 138 nr = get_user_pages(addr, nr, gup->flags, pages + i, 139 NULL); 140 break; 141 case PIN_FAST_BENCHMARK: 142 nr = pin_user_pages_fast(addr, nr, gup->flags, 143 pages + i); 144 break; 145 case PIN_BASIC_TEST: 146 nr = pin_user_pages(addr, nr, gup->flags, pages + i, 147 NULL); 148 break; 149 case PIN_LONGTERM_BENCHMARK: 150 nr = pin_user_pages(addr, nr, 151 gup->flags | FOLL_LONGTERM, 152 pages + i, NULL); 153 break; 154 case DUMP_USER_PAGES_TEST: 155 if (gup->flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) 156 nr = pin_user_pages(addr, nr, gup->flags, 157 pages + i, NULL); 158 else 159 nr = get_user_pages(addr, nr, gup->flags, 160 pages + i, NULL); 161 break; 162 default: 163 ret = -EINVAL; 164 goto unlock; 165 } 166 167 if (nr <= 0) 168 break; 169 i += nr; 170 } 171 end_time = ktime_get(); 172 173 /* Shifting the meaning of nr_pages: now it is actual number pinned: */ 174 nr_pages = i; 175 176 gup->get_delta_usec = ktime_us_delta(end_time, start_time); 177 gup->size = addr - gup->addr; 178 179 /* 180 * Take an un-benchmark-timed moment to verify DMA pinned 181 * state: print a warning if any non-dma-pinned pages are found: 182 */ 183 verify_dma_pinned(cmd, pages, nr_pages); 184 185 if (cmd == DUMP_USER_PAGES_TEST) 186 dump_pages_test(gup, pages, nr_pages); 187 188 start_time = ktime_get(); 189 190 put_back_pages(cmd, pages, nr_pages, gup->flags); 191 192 end_time = ktime_get(); 193 gup->put_delta_usec = ktime_us_delta(end_time, start_time); 194 195 unlock: 196 if (needs_mmap_lock) 197 mmap_read_unlock(current->mm); 198 free_pages: 199 kvfree(pages); 200 return ret; 201 } 202 203 static long gup_test_ioctl(struct file *filep, unsigned int cmd, 204 unsigned long arg) 205 { 206 struct gup_test gup; 207 int ret; 208 209 switch (cmd) { 210 case GUP_FAST_BENCHMARK: 211 case PIN_FAST_BENCHMARK: 212 case PIN_LONGTERM_BENCHMARK: 213 case GUP_BASIC_TEST: 214 case PIN_BASIC_TEST: 215 case DUMP_USER_PAGES_TEST: 216 break; 217 default: 218 return -EINVAL; 219 } 220 221 if (copy_from_user(&gup, (void __user *)arg, sizeof(gup))) 222 return -EFAULT; 223 224 ret = __gup_test_ioctl(cmd, &gup); 225 if (ret) 226 return ret; 227 228 if (copy_to_user((void __user *)arg, &gup, sizeof(gup))) 229 return -EFAULT; 230 231 return 0; 232 } 233 234 static const struct file_operations gup_test_fops = { 235 .open = nonseekable_open, 236 .unlocked_ioctl = gup_test_ioctl, 237 }; 238 239 static int __init gup_test_init(void) 240 { 241 debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL, 242 &gup_test_fops); 243 244 return 0; 245 } 246 247 late_initcall(gup_test_init); 248