1 /****************************************************************************** 2 * gntdev.c 3 * 4 * Device for accessing (in user-space) pages that have been granted by other 5 * domains. 6 * 7 * Copyright (c) 2006-2007, D G Murray. 8 * (c) 2009 Gerd Hoffmann <kraxel@redhat.com> 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 20 #undef DEBUG 21 22 #include <linux/module.h> 23 #include <linux/kernel.h> 24 #include <linux/init.h> 25 #include <linux/miscdevice.h> 26 #include <linux/fs.h> 27 #include <linux/mm.h> 28 #include <linux/mman.h> 29 #include <linux/mmu_notifier.h> 30 #include <linux/types.h> 31 #include <linux/uaccess.h> 32 #include <linux/sched.h> 33 #include <linux/spinlock.h> 34 #include <linux/slab.h> 35 36 #include <xen/xen.h> 37 #include <xen/grant_table.h> 38 #include <xen/gntdev.h> 39 #include <asm/xen/hypervisor.h> 40 #include <asm/xen/hypercall.h> 41 #include <asm/xen/page.h> 42 43 MODULE_LICENSE("GPL"); 44 MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, " 45 "Gerd Hoffmann <kraxel@redhat.com>"); 46 MODULE_DESCRIPTION("User-space granted page access driver"); 47 48 static int limit = 1024; 49 module_param(limit, int, 0644); 50 MODULE_PARM_DESC(limit, "Maximum number of grants that may be mapped at " 51 "once by a gntdev instance"); 52 53 struct gntdev_priv { 54 struct list_head maps; 55 uint32_t used; 56 uint32_t limit; 57 /* lock protects maps from concurrent changes */ 58 spinlock_t lock; 59 struct mm_struct *mm; 60 struct mmu_notifier mn; 61 }; 62 63 struct grant_map { 64 struct list_head next; 65 struct gntdev_priv *priv; 66 struct vm_area_struct *vma; 67 int index; 68 int count; 69 int flags; 70 int is_mapped; 71 struct ioctl_gntdev_grant_ref *grants; 72 struct gnttab_map_grant_ref *map_ops; 73 struct gnttab_unmap_grant_ref *unmap_ops; 74 struct page **pages; 75 }; 76 77 /* ------------------------------------------------------------------ */ 78 79 static void gntdev_print_maps(struct gntdev_priv *priv, 80 char *text, int text_index) 81 { 82 #ifdef DEBUG 83 struct grant_map *map; 84 85 pr_debug("maps list (priv %p, usage %d/%d)\n", 86 priv, priv->used, priv->limit); 87 88 list_for_each_entry(map, &priv->maps, next) 89 pr_debug(" index %2d, count %2d %s\n", 90 map->index, map->count, 91 map->index == text_index && text ? text : ""); 92 #endif 93 } 94 95 static struct grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count) 96 { 97 struct grant_map *add; 98 int i; 99 100 add = kzalloc(sizeof(struct grant_map), GFP_KERNEL); 101 if (NULL == add) 102 return NULL; 103 104 add->grants = kzalloc(sizeof(add->grants[0]) * count, GFP_KERNEL); 105 add->map_ops = kzalloc(sizeof(add->map_ops[0]) * count, GFP_KERNEL); 106 add->unmap_ops = kzalloc(sizeof(add->unmap_ops[0]) * count, GFP_KERNEL); 107 add->pages = kzalloc(sizeof(add->pages[0]) * count, GFP_KERNEL); 108 if (NULL == add->grants || 109 NULL == add->map_ops || 110 NULL == add->unmap_ops || 111 NULL == add->pages) 112 goto err; 113 114 for (i = 0; i < count; i++) { 115 add->pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); 116 if (add->pages[i] == NULL) 117 goto err; 118 } 119 120 add->index = 0; 121 add->count = count; 122 add->priv = priv; 123 124 if (add->count + priv->used > priv->limit) 125 goto err; 126 127 return add; 128 129 err: 130 if (add->pages) 131 for (i = 0; i < count; i++) { 132 if (add->pages[i]) 133 __free_page(add->pages[i]); 134 } 135 kfree(add->pages); 136 kfree(add->grants); 137 kfree(add->map_ops); 138 kfree(add->unmap_ops); 139 kfree(add); 140 return NULL; 141 } 142 143 static void gntdev_add_map(struct gntdev_priv *priv, struct grant_map *add) 144 { 145 struct grant_map *map; 146 147 list_for_each_entry(map, &priv->maps, next) { 148 if (add->index + add->count < map->index) { 149 list_add_tail(&add->next, &map->next); 150 goto done; 151 } 152 add->index = map->index + map->count; 153 } 154 list_add_tail(&add->next, &priv->maps); 155 156 done: 157 priv->used += add->count; 158 gntdev_print_maps(priv, "[new]", add->index); 159 } 160 161 static struct grant_map *gntdev_find_map_index(struct gntdev_priv *priv, 162 int index, int count) 163 { 164 struct grant_map *map; 165 166 list_for_each_entry(map, &priv->maps, next) { 167 if (map->index != index) 168 continue; 169 if (map->count != count) 170 continue; 171 return map; 172 } 173 return NULL; 174 } 175 176 static struct grant_map *gntdev_find_map_vaddr(struct gntdev_priv *priv, 177 unsigned long vaddr) 178 { 179 struct grant_map *map; 180 181 list_for_each_entry(map, &priv->maps, next) { 182 if (!map->vma) 183 continue; 184 if (vaddr < map->vma->vm_start) 185 continue; 186 if (vaddr >= map->vma->vm_end) 187 continue; 188 return map; 189 } 190 return NULL; 191 } 192 193 static int gntdev_del_map(struct grant_map *map) 194 { 195 int i; 196 197 if (map->vma) 198 return -EBUSY; 199 for (i = 0; i < map->count; i++) 200 if (map->unmap_ops[i].handle) 201 return -EBUSY; 202 203 map->priv->used -= map->count; 204 list_del(&map->next); 205 return 0; 206 } 207 208 static void gntdev_free_map(struct grant_map *map) 209 { 210 int i; 211 212 if (!map) 213 return; 214 215 if (map->pages) 216 for (i = 0; i < map->count; i++) { 217 if (map->pages[i]) 218 __free_page(map->pages[i]); 219 } 220 kfree(map->pages); 221 kfree(map->grants); 222 kfree(map->map_ops); 223 kfree(map->unmap_ops); 224 kfree(map); 225 } 226 227 /* ------------------------------------------------------------------ */ 228 229 static int find_grant_ptes(pte_t *pte, pgtable_t token, 230 unsigned long addr, void *data) 231 { 232 struct grant_map *map = data; 233 unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT; 234 u64 pte_maddr; 235 236 BUG_ON(pgnr >= map->count); 237 pte_maddr = arbitrary_virt_to_machine(pte).maddr; 238 239 gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, 240 GNTMAP_contains_pte | map->flags, 241 map->grants[pgnr].ref, 242 map->grants[pgnr].domid); 243 gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, 244 GNTMAP_contains_pte | map->flags, 245 0 /* handle */); 246 return 0; 247 } 248 249 static int map_grant_pages(struct grant_map *map) 250 { 251 int i, err = 0; 252 253 pr_debug("map %d+%d\n", map->index, map->count); 254 err = gnttab_map_refs(map->map_ops, map->pages, map->count); 255 if (err) 256 return err; 257 258 for (i = 0; i < map->count; i++) { 259 if (map->map_ops[i].status) 260 err = -EINVAL; 261 map->unmap_ops[i].handle = map->map_ops[i].handle; 262 } 263 return err; 264 } 265 266 static int unmap_grant_pages(struct grant_map *map, int offset, int pages) 267 { 268 int i, err = 0; 269 270 pr_debug("map %d+%d [%d+%d]\n", map->index, map->count, offset, pages); 271 err = gnttab_unmap_refs(map->unmap_ops + offset, map->pages, pages); 272 if (err) 273 return err; 274 275 for (i = 0; i < pages; i++) { 276 if (map->unmap_ops[offset+i].status) 277 err = -EINVAL; 278 map->unmap_ops[offset+i].handle = 0; 279 } 280 return err; 281 } 282 283 /* ------------------------------------------------------------------ */ 284 285 static void gntdev_vma_close(struct vm_area_struct *vma) 286 { 287 struct grant_map *map = vma->vm_private_data; 288 289 pr_debug("close %p\n", vma); 290 map->is_mapped = 0; 291 map->vma = NULL; 292 vma->vm_private_data = NULL; 293 } 294 295 static int gntdev_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 296 { 297 pr_debug("vaddr %p, pgoff %ld (shouldn't happen)\n", 298 vmf->virtual_address, vmf->pgoff); 299 vmf->flags = VM_FAULT_ERROR; 300 return 0; 301 } 302 303 static struct vm_operations_struct gntdev_vmops = { 304 .close = gntdev_vma_close, 305 .fault = gntdev_vma_fault, 306 }; 307 308 /* ------------------------------------------------------------------ */ 309 310 static void mn_invl_range_start(struct mmu_notifier *mn, 311 struct mm_struct *mm, 312 unsigned long start, unsigned long end) 313 { 314 struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); 315 struct grant_map *map; 316 unsigned long mstart, mend; 317 int err; 318 319 spin_lock(&priv->lock); 320 list_for_each_entry(map, &priv->maps, next) { 321 if (!map->vma) 322 continue; 323 if (!map->is_mapped) 324 continue; 325 if (map->vma->vm_start >= end) 326 continue; 327 if (map->vma->vm_end <= start) 328 continue; 329 mstart = max(start, map->vma->vm_start); 330 mend = min(end, map->vma->vm_end); 331 pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n", 332 map->index, map->count, 333 map->vma->vm_start, map->vma->vm_end, 334 start, end, mstart, mend); 335 err = unmap_grant_pages(map, 336 (mstart - map->vma->vm_start) >> PAGE_SHIFT, 337 (mend - mstart) >> PAGE_SHIFT); 338 WARN_ON(err); 339 } 340 spin_unlock(&priv->lock); 341 } 342 343 static void mn_invl_page(struct mmu_notifier *mn, 344 struct mm_struct *mm, 345 unsigned long address) 346 { 347 mn_invl_range_start(mn, mm, address, address + PAGE_SIZE); 348 } 349 350 static void mn_release(struct mmu_notifier *mn, 351 struct mm_struct *mm) 352 { 353 struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); 354 struct grant_map *map; 355 int err; 356 357 spin_lock(&priv->lock); 358 list_for_each_entry(map, &priv->maps, next) { 359 if (!map->vma) 360 continue; 361 pr_debug("map %d+%d (%lx %lx)\n", 362 map->index, map->count, 363 map->vma->vm_start, map->vma->vm_end); 364 err = unmap_grant_pages(map, /* offset */ 0, map->count); 365 WARN_ON(err); 366 } 367 spin_unlock(&priv->lock); 368 } 369 370 struct mmu_notifier_ops gntdev_mmu_ops = { 371 .release = mn_release, 372 .invalidate_page = mn_invl_page, 373 .invalidate_range_start = mn_invl_range_start, 374 }; 375 376 /* ------------------------------------------------------------------ */ 377 378 static int gntdev_open(struct inode *inode, struct file *flip) 379 { 380 struct gntdev_priv *priv; 381 int ret = 0; 382 383 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 384 if (!priv) 385 return -ENOMEM; 386 387 INIT_LIST_HEAD(&priv->maps); 388 spin_lock_init(&priv->lock); 389 priv->limit = limit; 390 391 priv->mm = get_task_mm(current); 392 if (!priv->mm) { 393 kfree(priv); 394 return -ENOMEM; 395 } 396 priv->mn.ops = &gntdev_mmu_ops; 397 ret = mmu_notifier_register(&priv->mn, priv->mm); 398 mmput(priv->mm); 399 400 if (ret) { 401 kfree(priv); 402 return ret; 403 } 404 405 flip->private_data = priv; 406 pr_debug("priv %p\n", priv); 407 408 return 0; 409 } 410 411 static int gntdev_release(struct inode *inode, struct file *flip) 412 { 413 struct gntdev_priv *priv = flip->private_data; 414 struct grant_map *map; 415 int err; 416 417 pr_debug("priv %p\n", priv); 418 419 spin_lock(&priv->lock); 420 while (!list_empty(&priv->maps)) { 421 map = list_entry(priv->maps.next, struct grant_map, next); 422 err = gntdev_del_map(map); 423 if (WARN_ON(err)) 424 gntdev_free_map(map); 425 426 } 427 spin_unlock(&priv->lock); 428 429 mmu_notifier_unregister(&priv->mn, priv->mm); 430 kfree(priv); 431 return 0; 432 } 433 434 static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, 435 struct ioctl_gntdev_map_grant_ref __user *u) 436 { 437 struct ioctl_gntdev_map_grant_ref op; 438 struct grant_map *map; 439 int err; 440 441 if (copy_from_user(&op, u, sizeof(op)) != 0) 442 return -EFAULT; 443 pr_debug("priv %p, add %d\n", priv, op.count); 444 if (unlikely(op.count <= 0)) 445 return -EINVAL; 446 if (unlikely(op.count > priv->limit)) 447 return -EINVAL; 448 449 err = -ENOMEM; 450 map = gntdev_alloc_map(priv, op.count); 451 if (!map) 452 return err; 453 if (copy_from_user(map->grants, &u->refs, 454 sizeof(map->grants[0]) * op.count) != 0) { 455 gntdev_free_map(map); 456 return err; 457 } 458 459 spin_lock(&priv->lock); 460 gntdev_add_map(priv, map); 461 op.index = map->index << PAGE_SHIFT; 462 spin_unlock(&priv->lock); 463 464 if (copy_to_user(u, &op, sizeof(op)) != 0) { 465 spin_lock(&priv->lock); 466 gntdev_del_map(map); 467 spin_unlock(&priv->lock); 468 gntdev_free_map(map); 469 return err; 470 } 471 return 0; 472 } 473 474 static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, 475 struct ioctl_gntdev_unmap_grant_ref __user *u) 476 { 477 struct ioctl_gntdev_unmap_grant_ref op; 478 struct grant_map *map; 479 int err = -ENOENT; 480 481 if (copy_from_user(&op, u, sizeof(op)) != 0) 482 return -EFAULT; 483 pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count); 484 485 spin_lock(&priv->lock); 486 map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count); 487 if (map) 488 err = gntdev_del_map(map); 489 spin_unlock(&priv->lock); 490 if (!err) 491 gntdev_free_map(map); 492 return err; 493 } 494 495 static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv, 496 struct ioctl_gntdev_get_offset_for_vaddr __user *u) 497 { 498 struct ioctl_gntdev_get_offset_for_vaddr op; 499 struct grant_map *map; 500 501 if (copy_from_user(&op, u, sizeof(op)) != 0) 502 return -EFAULT; 503 pr_debug("priv %p, offset for vaddr %lx\n", priv, (unsigned long)op.vaddr); 504 505 spin_lock(&priv->lock); 506 map = gntdev_find_map_vaddr(priv, op.vaddr); 507 if (map == NULL || 508 map->vma->vm_start != op.vaddr) { 509 spin_unlock(&priv->lock); 510 return -EINVAL; 511 } 512 op.offset = map->index << PAGE_SHIFT; 513 op.count = map->count; 514 spin_unlock(&priv->lock); 515 516 if (copy_to_user(u, &op, sizeof(op)) != 0) 517 return -EFAULT; 518 return 0; 519 } 520 521 static long gntdev_ioctl_set_max_grants(struct gntdev_priv *priv, 522 struct ioctl_gntdev_set_max_grants __user *u) 523 { 524 struct ioctl_gntdev_set_max_grants op; 525 526 if (copy_from_user(&op, u, sizeof(op)) != 0) 527 return -EFAULT; 528 pr_debug("priv %p, limit %d\n", priv, op.count); 529 if (op.count > limit) 530 return -E2BIG; 531 532 spin_lock(&priv->lock); 533 priv->limit = op.count; 534 spin_unlock(&priv->lock); 535 return 0; 536 } 537 538 static long gntdev_ioctl(struct file *flip, 539 unsigned int cmd, unsigned long arg) 540 { 541 struct gntdev_priv *priv = flip->private_data; 542 void __user *ptr = (void __user *)arg; 543 544 switch (cmd) { 545 case IOCTL_GNTDEV_MAP_GRANT_REF: 546 return gntdev_ioctl_map_grant_ref(priv, ptr); 547 548 case IOCTL_GNTDEV_UNMAP_GRANT_REF: 549 return gntdev_ioctl_unmap_grant_ref(priv, ptr); 550 551 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR: 552 return gntdev_ioctl_get_offset_for_vaddr(priv, ptr); 553 554 case IOCTL_GNTDEV_SET_MAX_GRANTS: 555 return gntdev_ioctl_set_max_grants(priv, ptr); 556 557 default: 558 pr_debug("priv %p, unknown cmd %x\n", priv, cmd); 559 return -ENOIOCTLCMD; 560 } 561 562 return 0; 563 } 564 565 static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) 566 { 567 struct gntdev_priv *priv = flip->private_data; 568 int index = vma->vm_pgoff; 569 int count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 570 struct grant_map *map; 571 int err = -EINVAL; 572 573 if ((vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_SHARED)) 574 return -EINVAL; 575 576 pr_debug("map %d+%d at %lx (pgoff %lx)\n", 577 index, count, vma->vm_start, vma->vm_pgoff); 578 579 spin_lock(&priv->lock); 580 map = gntdev_find_map_index(priv, index, count); 581 if (!map) 582 goto unlock_out; 583 if (map->vma) 584 goto unlock_out; 585 if (priv->mm != vma->vm_mm) { 586 printk(KERN_WARNING "Huh? Other mm?\n"); 587 goto unlock_out; 588 } 589 590 vma->vm_ops = &gntdev_vmops; 591 592 vma->vm_flags |= VM_RESERVED|VM_DONTCOPY|VM_DONTEXPAND|VM_PFNMAP; 593 594 vma->vm_private_data = map; 595 map->vma = vma; 596 597 map->flags = GNTMAP_host_map | GNTMAP_application_map; 598 if (!(vma->vm_flags & VM_WRITE)) 599 map->flags |= GNTMAP_readonly; 600 601 spin_unlock(&priv->lock); 602 603 err = apply_to_page_range(vma->vm_mm, vma->vm_start, 604 vma->vm_end - vma->vm_start, 605 find_grant_ptes, map); 606 if (err) { 607 printk(KERN_WARNING "find_grant_ptes() failure.\n"); 608 return err; 609 } 610 611 err = map_grant_pages(map); 612 if (err) { 613 printk(KERN_WARNING "map_grant_pages() failure.\n"); 614 return err; 615 } 616 617 map->is_mapped = 1; 618 619 return 0; 620 621 unlock_out: 622 spin_unlock(&priv->lock); 623 return err; 624 } 625 626 static const struct file_operations gntdev_fops = { 627 .owner = THIS_MODULE, 628 .open = gntdev_open, 629 .release = gntdev_release, 630 .mmap = gntdev_mmap, 631 .unlocked_ioctl = gntdev_ioctl 632 }; 633 634 static struct miscdevice gntdev_miscdev = { 635 .minor = MISC_DYNAMIC_MINOR, 636 .name = "xen/gntdev", 637 .fops = &gntdev_fops, 638 }; 639 640 /* ------------------------------------------------------------------ */ 641 642 static int __init gntdev_init(void) 643 { 644 int err; 645 646 if (!xen_domain()) 647 return -ENODEV; 648 649 err = misc_register(&gntdev_miscdev); 650 if (err != 0) { 651 printk(KERN_ERR "Could not register gntdev device\n"); 652 return err; 653 } 654 return 0; 655 } 656 657 static void __exit gntdev_exit(void) 658 { 659 misc_deregister(&gntdev_miscdev); 660 } 661 662 module_init(gntdev_init); 663 module_exit(gntdev_exit); 664 665 /* ------------------------------------------------------------------ */ 666