1 // SPDX-License-Identifier: GPL-2.0-only 2 /****************************************************************************** 3 * privcmd.c 4 * 5 * Interface to privileged domain-0 commands. 6 * 7 * Copyright (c) 2002-2004, K A Fraser, B Dragovic 8 */ 9 10 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 #include <linux/sched.h> 15 #include <linux/slab.h> 16 #include <linux/string.h> 17 #include <linux/errno.h> 18 #include <linux/mm.h> 19 #include <linux/mman.h> 20 #include <linux/uaccess.h> 21 #include <linux/swap.h> 22 #include <linux/highmem.h> 23 #include <linux/pagemap.h> 24 #include <linux/seq_file.h> 25 #include <linux/miscdevice.h> 26 #include <linux/moduleparam.h> 27 28 #include <asm/pgalloc.h> 29 #include <asm/pgtable.h> 30 #include <asm/tlb.h> 31 #include <asm/xen/hypervisor.h> 32 #include <asm/xen/hypercall.h> 33 34 #include <xen/xen.h> 35 #include <xen/privcmd.h> 36 #include <xen/interface/xen.h> 37 #include <xen/interface/memory.h> 38 #include <xen/interface/hvm/dm_op.h> 39 #include <xen/features.h> 40 #include <xen/page.h> 41 #include <xen/xen-ops.h> 42 #include <xen/balloon.h> 43 44 #include "privcmd.h" 45 46 MODULE_LICENSE("GPL"); 47 48 #define PRIV_VMA_LOCKED ((void *)1) 49 50 static unsigned int privcmd_dm_op_max_num = 16; 51 module_param_named(dm_op_max_nr_bufs, privcmd_dm_op_max_num, uint, 0644); 52 MODULE_PARM_DESC(dm_op_max_nr_bufs, 53 "Maximum number of buffers per dm_op hypercall"); 54 55 static unsigned int privcmd_dm_op_buf_max_size = 4096; 56 module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint, 57 0644); 58 MODULE_PARM_DESC(dm_op_buf_max_size, 59 "Maximum size of a dm_op hypercall buffer"); 60 61 struct privcmd_data { 62 domid_t domid; 63 }; 64 65 static int privcmd_vma_range_is_mapped( 66 struct vm_area_struct *vma, 67 unsigned long addr, 68 unsigned long nr_pages); 69 70 static long privcmd_ioctl_hypercall(struct file *file, void __user *udata) 71 { 72 struct privcmd_data *data = file->private_data; 73 struct privcmd_hypercall hypercall; 74 long ret; 75 76 /* Disallow arbitrary hypercalls if restricted */ 77 if (data->domid != DOMID_INVALID) 78 return -EPERM; 79 80 if (copy_from_user(&hypercall, udata, sizeof(hypercall))) 81 return -EFAULT; 82 83 xen_preemptible_hcall_begin(); 84 ret = privcmd_call(hypercall.op, 85 hypercall.arg[0], hypercall.arg[1], 86 hypercall.arg[2], hypercall.arg[3], 87 hypercall.arg[4]); 88 xen_preemptible_hcall_end(); 89 90 return ret; 91 } 92 93 static void free_page_list(struct list_head *pages) 94 { 95 struct page *p, *n; 96 97 list_for_each_entry_safe(p, n, pages, lru) 98 __free_page(p); 99 100 INIT_LIST_HEAD(pages); 101 } 102 103 /* 104 * Given an array of items in userspace, return a list of pages 105 * containing the data. If copying fails, either because of memory 106 * allocation failure or a problem reading user memory, return an 107 * error code; its up to the caller to dispose of any partial list. 108 */ 109 static int gather_array(struct list_head *pagelist, 110 unsigned nelem, size_t size, 111 const void __user *data) 112 { 113 unsigned pageidx; 114 void *pagedata; 115 int ret; 116 117 if (size > PAGE_SIZE) 118 return 0; 119 120 pageidx = PAGE_SIZE; 121 pagedata = NULL; /* quiet, gcc */ 122 while (nelem--) { 123 if (pageidx > PAGE_SIZE-size) { 124 struct page *page = alloc_page(GFP_KERNEL); 125 126 ret = -ENOMEM; 127 if (page == NULL) 128 goto fail; 129 130 pagedata = page_address(page); 131 132 list_add_tail(&page->lru, pagelist); 133 pageidx = 0; 134 } 135 136 ret = -EFAULT; 137 if (copy_from_user(pagedata + pageidx, data, size)) 138 goto fail; 139 140 data += size; 141 pageidx += size; 142 } 143 144 ret = 0; 145 146 fail: 147 return ret; 148 } 149 150 /* 151 * Call function "fn" on each element of the array fragmented 152 * over a list of pages. 153 */ 154 static int traverse_pages(unsigned nelem, size_t size, 155 struct list_head *pos, 156 int (*fn)(void *data, void *state), 157 void *state) 158 { 159 void *pagedata; 160 unsigned pageidx; 161 int ret = 0; 162 163 BUG_ON(size > PAGE_SIZE); 164 165 pageidx = PAGE_SIZE; 166 pagedata = NULL; /* hush, gcc */ 167 168 while (nelem--) { 169 if (pageidx > PAGE_SIZE-size) { 170 struct page *page; 171 pos = pos->next; 172 page = list_entry(pos, struct page, lru); 173 pagedata = page_address(page); 174 pageidx = 0; 175 } 176 177 ret = (*fn)(pagedata + pageidx, state); 178 if (ret) 179 break; 180 pageidx += size; 181 } 182 183 return ret; 184 } 185 186 /* 187 * Similar to traverse_pages, but use each page as a "block" of 188 * data to be processed as one unit. 189 */ 190 static int traverse_pages_block(unsigned nelem, size_t size, 191 struct list_head *pos, 192 int (*fn)(void *data, int nr, void *state), 193 void *state) 194 { 195 void *pagedata; 196 int ret = 0; 197 198 BUG_ON(size > PAGE_SIZE); 199 200 while (nelem) { 201 int nr = (PAGE_SIZE/size); 202 struct page *page; 203 if (nr > nelem) 204 nr = nelem; 205 pos = pos->next; 206 page = list_entry(pos, struct page, lru); 207 pagedata = page_address(page); 208 ret = (*fn)(pagedata, nr, state); 209 if (ret) 210 break; 211 nelem -= nr; 212 } 213 214 return ret; 215 } 216 217 struct mmap_gfn_state { 218 unsigned long va; 219 struct vm_area_struct *vma; 220 domid_t domain; 221 }; 222 223 static int mmap_gfn_range(void *data, void *state) 224 { 225 struct privcmd_mmap_entry *msg = data; 226 struct mmap_gfn_state *st = state; 227 struct vm_area_struct *vma = st->vma; 228 int rc; 229 230 /* Do not allow range to wrap the address space. */ 231 if ((msg->npages > (LONG_MAX >> PAGE_SHIFT)) || 232 ((unsigned long)(msg->npages << PAGE_SHIFT) >= -st->va)) 233 return -EINVAL; 234 235 /* Range chunks must be contiguous in va space. */ 236 if ((msg->va != st->va) || 237 ((msg->va+(msg->npages<<PAGE_SHIFT)) > vma->vm_end)) 238 return -EINVAL; 239 240 rc = xen_remap_domain_gfn_range(vma, 241 msg->va & PAGE_MASK, 242 msg->mfn, msg->npages, 243 vma->vm_page_prot, 244 st->domain, NULL); 245 if (rc < 0) 246 return rc; 247 248 st->va += msg->npages << PAGE_SHIFT; 249 250 return 0; 251 } 252 253 static long privcmd_ioctl_mmap(struct file *file, void __user *udata) 254 { 255 struct privcmd_data *data = file->private_data; 256 struct privcmd_mmap mmapcmd; 257 struct mm_struct *mm = current->mm; 258 struct vm_area_struct *vma; 259 int rc; 260 LIST_HEAD(pagelist); 261 struct mmap_gfn_state state; 262 263 /* We only support privcmd_ioctl_mmap_batch for auto translated. */ 264 if (xen_feature(XENFEAT_auto_translated_physmap)) 265 return -ENOSYS; 266 267 if (copy_from_user(&mmapcmd, udata, sizeof(mmapcmd))) 268 return -EFAULT; 269 270 /* If restriction is in place, check the domid matches */ 271 if (data->domid != DOMID_INVALID && data->domid != mmapcmd.dom) 272 return -EPERM; 273 274 rc = gather_array(&pagelist, 275 mmapcmd.num, sizeof(struct privcmd_mmap_entry), 276 mmapcmd.entry); 277 278 if (rc || list_empty(&pagelist)) 279 goto out; 280 281 down_write(&mm->mmap_sem); 282 283 { 284 struct page *page = list_first_entry(&pagelist, 285 struct page, lru); 286 struct privcmd_mmap_entry *msg = page_address(page); 287 288 vma = find_vma(mm, msg->va); 289 rc = -EINVAL; 290 291 if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) 292 goto out_up; 293 vma->vm_private_data = PRIV_VMA_LOCKED; 294 } 295 296 state.va = vma->vm_start; 297 state.vma = vma; 298 state.domain = mmapcmd.dom; 299 300 rc = traverse_pages(mmapcmd.num, sizeof(struct privcmd_mmap_entry), 301 &pagelist, 302 mmap_gfn_range, &state); 303 304 305 out_up: 306 up_write(&mm->mmap_sem); 307 308 out: 309 free_page_list(&pagelist); 310 311 return rc; 312 } 313 314 struct mmap_batch_state { 315 domid_t domain; 316 unsigned long va; 317 struct vm_area_struct *vma; 318 int index; 319 /* A tristate: 320 * 0 for no errors 321 * 1 if at least one error has happened (and no 322 * -ENOENT errors have happened) 323 * -ENOENT if at least 1 -ENOENT has happened. 324 */ 325 int global_error; 326 int version; 327 328 /* User-space gfn array to store errors in the second pass for V1. */ 329 xen_pfn_t __user *user_gfn; 330 /* User-space int array to store errors in the second pass for V2. */ 331 int __user *user_err; 332 }; 333 334 /* auto translated dom0 note: if domU being created is PV, then gfn is 335 * mfn(addr on bus). If it's auto xlated, then gfn is pfn (input to HAP). 336 */ 337 static int mmap_batch_fn(void *data, int nr, void *state) 338 { 339 xen_pfn_t *gfnp = data; 340 struct mmap_batch_state *st = state; 341 struct vm_area_struct *vma = st->vma; 342 struct page **pages = vma->vm_private_data; 343 struct page **cur_pages = NULL; 344 int ret; 345 346 if (xen_feature(XENFEAT_auto_translated_physmap)) 347 cur_pages = &pages[st->index]; 348 349 BUG_ON(nr < 0); 350 ret = xen_remap_domain_gfn_array(st->vma, st->va & PAGE_MASK, gfnp, nr, 351 (int *)gfnp, st->vma->vm_page_prot, 352 st->domain, cur_pages); 353 354 /* Adjust the global_error? */ 355 if (ret != nr) { 356 if (ret == -ENOENT) 357 st->global_error = -ENOENT; 358 else { 359 /* Record that at least one error has happened. */ 360 if (st->global_error == 0) 361 st->global_error = 1; 362 } 363 } 364 st->va += XEN_PAGE_SIZE * nr; 365 st->index += nr / XEN_PFN_PER_PAGE; 366 367 return 0; 368 } 369 370 static int mmap_return_error(int err, struct mmap_batch_state *st) 371 { 372 int ret; 373 374 if (st->version == 1) { 375 if (err) { 376 xen_pfn_t gfn; 377 378 ret = get_user(gfn, st->user_gfn); 379 if (ret < 0) 380 return ret; 381 /* 382 * V1 encodes the error codes in the 32bit top 383 * nibble of the gfn (with its known 384 * limitations vis-a-vis 64 bit callers). 385 */ 386 gfn |= (err == -ENOENT) ? 387 PRIVCMD_MMAPBATCH_PAGED_ERROR : 388 PRIVCMD_MMAPBATCH_MFN_ERROR; 389 return __put_user(gfn, st->user_gfn++); 390 } else 391 st->user_gfn++; 392 } else { /* st->version == 2 */ 393 if (err) 394 return __put_user(err, st->user_err++); 395 else 396 st->user_err++; 397 } 398 399 return 0; 400 } 401 402 static int mmap_return_errors(void *data, int nr, void *state) 403 { 404 struct mmap_batch_state *st = state; 405 int *errs = data; 406 int i; 407 int ret; 408 409 for (i = 0; i < nr; i++) { 410 ret = mmap_return_error(errs[i], st); 411 if (ret < 0) 412 return ret; 413 } 414 return 0; 415 } 416 417 /* Allocate pfns that are then mapped with gfns from foreign domid. Update 418 * the vma with the page info to use later. 419 * Returns: 0 if success, otherwise -errno 420 */ 421 static int alloc_empty_pages(struct vm_area_struct *vma, int numpgs) 422 { 423 int rc; 424 struct page **pages; 425 426 pages = kcalloc(numpgs, sizeof(pages[0]), GFP_KERNEL); 427 if (pages == NULL) 428 return -ENOMEM; 429 430 rc = alloc_xenballooned_pages(numpgs, pages); 431 if (rc != 0) { 432 pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__, 433 numpgs, rc); 434 kfree(pages); 435 return -ENOMEM; 436 } 437 BUG_ON(vma->vm_private_data != NULL); 438 vma->vm_private_data = pages; 439 440 return 0; 441 } 442 443 static const struct vm_operations_struct privcmd_vm_ops; 444 445 static long privcmd_ioctl_mmap_batch( 446 struct file *file, void __user *udata, int version) 447 { 448 struct privcmd_data *data = file->private_data; 449 int ret; 450 struct privcmd_mmapbatch_v2 m; 451 struct mm_struct *mm = current->mm; 452 struct vm_area_struct *vma; 453 unsigned long nr_pages; 454 LIST_HEAD(pagelist); 455 struct mmap_batch_state state; 456 457 switch (version) { 458 case 1: 459 if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch))) 460 return -EFAULT; 461 /* Returns per-frame error in m.arr. */ 462 m.err = NULL; 463 if (!access_ok(m.arr, m.num * sizeof(*m.arr))) 464 return -EFAULT; 465 break; 466 case 2: 467 if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2))) 468 return -EFAULT; 469 /* Returns per-frame error code in m.err. */ 470 if (!access_ok(m.err, m.num * (sizeof(*m.err)))) 471 return -EFAULT; 472 break; 473 default: 474 return -EINVAL; 475 } 476 477 /* If restriction is in place, check the domid matches */ 478 if (data->domid != DOMID_INVALID && data->domid != m.dom) 479 return -EPERM; 480 481 nr_pages = DIV_ROUND_UP(m.num, XEN_PFN_PER_PAGE); 482 if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT))) 483 return -EINVAL; 484 485 ret = gather_array(&pagelist, m.num, sizeof(xen_pfn_t), m.arr); 486 487 if (ret) 488 goto out; 489 if (list_empty(&pagelist)) { 490 ret = -EINVAL; 491 goto out; 492 } 493 494 if (version == 2) { 495 /* Zero error array now to only copy back actual errors. */ 496 if (clear_user(m.err, sizeof(int) * m.num)) { 497 ret = -EFAULT; 498 goto out; 499 } 500 } 501 502 down_write(&mm->mmap_sem); 503 504 vma = find_vma(mm, m.addr); 505 if (!vma || 506 vma->vm_ops != &privcmd_vm_ops) { 507 ret = -EINVAL; 508 goto out_unlock; 509 } 510 511 /* 512 * Caller must either: 513 * 514 * Map the whole VMA range, which will also allocate all the 515 * pages required for the auto_translated_physmap case. 516 * 517 * Or 518 * 519 * Map unmapped holes left from a previous map attempt (e.g., 520 * because those foreign frames were previously paged out). 521 */ 522 if (vma->vm_private_data == NULL) { 523 if (m.addr != vma->vm_start || 524 m.addr + (nr_pages << PAGE_SHIFT) != vma->vm_end) { 525 ret = -EINVAL; 526 goto out_unlock; 527 } 528 if (xen_feature(XENFEAT_auto_translated_physmap)) { 529 ret = alloc_empty_pages(vma, nr_pages); 530 if (ret < 0) 531 goto out_unlock; 532 } else 533 vma->vm_private_data = PRIV_VMA_LOCKED; 534 } else { 535 if (m.addr < vma->vm_start || 536 m.addr + (nr_pages << PAGE_SHIFT) > vma->vm_end) { 537 ret = -EINVAL; 538 goto out_unlock; 539 } 540 if (privcmd_vma_range_is_mapped(vma, m.addr, nr_pages)) { 541 ret = -EINVAL; 542 goto out_unlock; 543 } 544 } 545 546 state.domain = m.dom; 547 state.vma = vma; 548 state.va = m.addr; 549 state.index = 0; 550 state.global_error = 0; 551 state.version = version; 552 553 BUILD_BUG_ON(((PAGE_SIZE / sizeof(xen_pfn_t)) % XEN_PFN_PER_PAGE) != 0); 554 /* mmap_batch_fn guarantees ret == 0 */ 555 BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t), 556 &pagelist, mmap_batch_fn, &state)); 557 558 up_write(&mm->mmap_sem); 559 560 if (state.global_error) { 561 /* Write back errors in second pass. */ 562 state.user_gfn = (xen_pfn_t *)m.arr; 563 state.user_err = m.err; 564 ret = traverse_pages_block(m.num, sizeof(xen_pfn_t), 565 &pagelist, mmap_return_errors, &state); 566 } else 567 ret = 0; 568 569 /* If we have not had any EFAULT-like global errors then set the global 570 * error to -ENOENT if necessary. */ 571 if ((ret == 0) && (state.global_error == -ENOENT)) 572 ret = -ENOENT; 573 574 out: 575 free_page_list(&pagelist); 576 return ret; 577 578 out_unlock: 579 up_write(&mm->mmap_sem); 580 goto out; 581 } 582 583 static int lock_pages( 584 struct privcmd_dm_op_buf kbufs[], unsigned int num, 585 struct page *pages[], unsigned int nr_pages) 586 { 587 unsigned int i; 588 589 for (i = 0; i < num; i++) { 590 unsigned int requested; 591 int pinned; 592 593 requested = DIV_ROUND_UP( 594 offset_in_page(kbufs[i].uptr) + kbufs[i].size, 595 PAGE_SIZE); 596 if (requested > nr_pages) 597 return -ENOSPC; 598 599 pinned = get_user_pages_fast( 600 (unsigned long) kbufs[i].uptr, 601 requested, FOLL_WRITE, pages); 602 if (pinned < 0) 603 return pinned; 604 605 nr_pages -= pinned; 606 pages += pinned; 607 } 608 609 return 0; 610 } 611 612 static void unlock_pages(struct page *pages[], unsigned int nr_pages) 613 { 614 unsigned int i; 615 616 if (!pages) 617 return; 618 619 for (i = 0; i < nr_pages; i++) { 620 if (pages[i]) 621 put_page(pages[i]); 622 } 623 } 624 625 static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) 626 { 627 struct privcmd_data *data = file->private_data; 628 struct privcmd_dm_op kdata; 629 struct privcmd_dm_op_buf *kbufs; 630 unsigned int nr_pages = 0; 631 struct page **pages = NULL; 632 struct xen_dm_op_buf *xbufs = NULL; 633 unsigned int i; 634 long rc; 635 636 if (copy_from_user(&kdata, udata, sizeof(kdata))) 637 return -EFAULT; 638 639 /* If restriction is in place, check the domid matches */ 640 if (data->domid != DOMID_INVALID && data->domid != kdata.dom) 641 return -EPERM; 642 643 if (kdata.num == 0) 644 return 0; 645 646 if (kdata.num > privcmd_dm_op_max_num) 647 return -E2BIG; 648 649 kbufs = kcalloc(kdata.num, sizeof(*kbufs), GFP_KERNEL); 650 if (!kbufs) 651 return -ENOMEM; 652 653 if (copy_from_user(kbufs, kdata.ubufs, 654 sizeof(*kbufs) * kdata.num)) { 655 rc = -EFAULT; 656 goto out; 657 } 658 659 for (i = 0; i < kdata.num; i++) { 660 if (kbufs[i].size > privcmd_dm_op_buf_max_size) { 661 rc = -E2BIG; 662 goto out; 663 } 664 665 if (!access_ok(kbufs[i].uptr, 666 kbufs[i].size)) { 667 rc = -EFAULT; 668 goto out; 669 } 670 671 nr_pages += DIV_ROUND_UP( 672 offset_in_page(kbufs[i].uptr) + kbufs[i].size, 673 PAGE_SIZE); 674 } 675 676 pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); 677 if (!pages) { 678 rc = -ENOMEM; 679 goto out; 680 } 681 682 xbufs = kcalloc(kdata.num, sizeof(*xbufs), GFP_KERNEL); 683 if (!xbufs) { 684 rc = -ENOMEM; 685 goto out; 686 } 687 688 rc = lock_pages(kbufs, kdata.num, pages, nr_pages); 689 if (rc) 690 goto out; 691 692 for (i = 0; i < kdata.num; i++) { 693 set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr); 694 xbufs[i].size = kbufs[i].size; 695 } 696 697 xen_preemptible_hcall_begin(); 698 rc = HYPERVISOR_dm_op(kdata.dom, kdata.num, xbufs); 699 xen_preemptible_hcall_end(); 700 701 out: 702 unlock_pages(pages, nr_pages); 703 kfree(xbufs); 704 kfree(pages); 705 kfree(kbufs); 706 707 return rc; 708 } 709 710 static long privcmd_ioctl_restrict(struct file *file, void __user *udata) 711 { 712 struct privcmd_data *data = file->private_data; 713 domid_t dom; 714 715 if (copy_from_user(&dom, udata, sizeof(dom))) 716 return -EFAULT; 717 718 /* Set restriction to the specified domain, or check it matches */ 719 if (data->domid == DOMID_INVALID) 720 data->domid = dom; 721 else if (data->domid != dom) 722 return -EINVAL; 723 724 return 0; 725 } 726 727 struct remap_pfn { 728 struct mm_struct *mm; 729 struct page **pages; 730 pgprot_t prot; 731 unsigned long i; 732 }; 733 734 static int remap_pfn_fn(pte_t *ptep, pgtable_t token, unsigned long addr, 735 void *data) 736 { 737 struct remap_pfn *r = data; 738 struct page *page = r->pages[r->i]; 739 pte_t pte = pte_mkspecial(pfn_pte(page_to_pfn(page), r->prot)); 740 741 set_pte_at(r->mm, addr, ptep, pte); 742 r->i++; 743 744 return 0; 745 } 746 747 static long privcmd_ioctl_mmap_resource(struct file *file, void __user *udata) 748 { 749 struct privcmd_data *data = file->private_data; 750 struct mm_struct *mm = current->mm; 751 struct vm_area_struct *vma; 752 struct privcmd_mmap_resource kdata; 753 xen_pfn_t *pfns = NULL; 754 struct xen_mem_acquire_resource xdata; 755 int rc; 756 757 if (copy_from_user(&kdata, udata, sizeof(kdata))) 758 return -EFAULT; 759 760 /* If restriction is in place, check the domid matches */ 761 if (data->domid != DOMID_INVALID && data->domid != kdata.dom) 762 return -EPERM; 763 764 down_write(&mm->mmap_sem); 765 766 vma = find_vma(mm, kdata.addr); 767 if (!vma || vma->vm_ops != &privcmd_vm_ops) { 768 rc = -EINVAL; 769 goto out; 770 } 771 772 pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL); 773 if (!pfns) { 774 rc = -ENOMEM; 775 goto out; 776 } 777 778 if (xen_feature(XENFEAT_auto_translated_physmap)) { 779 unsigned int nr = DIV_ROUND_UP(kdata.num, XEN_PFN_PER_PAGE); 780 struct page **pages; 781 unsigned int i; 782 783 rc = alloc_empty_pages(vma, nr); 784 if (rc < 0) 785 goto out; 786 787 pages = vma->vm_private_data; 788 for (i = 0; i < kdata.num; i++) { 789 xen_pfn_t pfn = 790 page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]); 791 792 pfns[i] = pfn + (i % XEN_PFN_PER_PAGE); 793 } 794 } else 795 vma->vm_private_data = PRIV_VMA_LOCKED; 796 797 memset(&xdata, 0, sizeof(xdata)); 798 xdata.domid = kdata.dom; 799 xdata.type = kdata.type; 800 xdata.id = kdata.id; 801 xdata.frame = kdata.idx; 802 xdata.nr_frames = kdata.num; 803 set_xen_guest_handle(xdata.frame_list, pfns); 804 805 xen_preemptible_hcall_begin(); 806 rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, &xdata); 807 xen_preemptible_hcall_end(); 808 809 if (rc) 810 goto out; 811 812 if (xen_feature(XENFEAT_auto_translated_physmap)) { 813 struct remap_pfn r = { 814 .mm = vma->vm_mm, 815 .pages = vma->vm_private_data, 816 .prot = vma->vm_page_prot, 817 }; 818 819 rc = apply_to_page_range(r.mm, kdata.addr, 820 kdata.num << PAGE_SHIFT, 821 remap_pfn_fn, &r); 822 } else { 823 unsigned int domid = 824 (xdata.flags & XENMEM_rsrc_acq_caller_owned) ? 825 DOMID_SELF : kdata.dom; 826 int num; 827 828 num = xen_remap_domain_mfn_array(vma, 829 kdata.addr & PAGE_MASK, 830 pfns, kdata.num, (int *)pfns, 831 vma->vm_page_prot, 832 domid, 833 vma->vm_private_data); 834 if (num < 0) 835 rc = num; 836 else if (num != kdata.num) { 837 unsigned int i; 838 839 for (i = 0; i < num; i++) { 840 rc = pfns[i]; 841 if (rc < 0) 842 break; 843 } 844 } else 845 rc = 0; 846 } 847 848 out: 849 up_write(&mm->mmap_sem); 850 kfree(pfns); 851 852 return rc; 853 } 854 855 static long privcmd_ioctl(struct file *file, 856 unsigned int cmd, unsigned long data) 857 { 858 int ret = -ENOTTY; 859 void __user *udata = (void __user *) data; 860 861 switch (cmd) { 862 case IOCTL_PRIVCMD_HYPERCALL: 863 ret = privcmd_ioctl_hypercall(file, udata); 864 break; 865 866 case IOCTL_PRIVCMD_MMAP: 867 ret = privcmd_ioctl_mmap(file, udata); 868 break; 869 870 case IOCTL_PRIVCMD_MMAPBATCH: 871 ret = privcmd_ioctl_mmap_batch(file, udata, 1); 872 break; 873 874 case IOCTL_PRIVCMD_MMAPBATCH_V2: 875 ret = privcmd_ioctl_mmap_batch(file, udata, 2); 876 break; 877 878 case IOCTL_PRIVCMD_DM_OP: 879 ret = privcmd_ioctl_dm_op(file, udata); 880 break; 881 882 case IOCTL_PRIVCMD_RESTRICT: 883 ret = privcmd_ioctl_restrict(file, udata); 884 break; 885 886 case IOCTL_PRIVCMD_MMAP_RESOURCE: 887 ret = privcmd_ioctl_mmap_resource(file, udata); 888 break; 889 890 default: 891 break; 892 } 893 894 return ret; 895 } 896 897 static int privcmd_open(struct inode *ino, struct file *file) 898 { 899 struct privcmd_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 900 901 if (!data) 902 return -ENOMEM; 903 904 /* DOMID_INVALID implies no restriction */ 905 data->domid = DOMID_INVALID; 906 907 file->private_data = data; 908 return 0; 909 } 910 911 static int privcmd_release(struct inode *ino, struct file *file) 912 { 913 struct privcmd_data *data = file->private_data; 914 915 kfree(data); 916 return 0; 917 } 918 919 static void privcmd_close(struct vm_area_struct *vma) 920 { 921 struct page **pages = vma->vm_private_data; 922 int numpgs = vma_pages(vma); 923 int numgfns = (vma->vm_end - vma->vm_start) >> XEN_PAGE_SHIFT; 924 int rc; 925 926 if (!xen_feature(XENFEAT_auto_translated_physmap) || !numpgs || !pages) 927 return; 928 929 rc = xen_unmap_domain_gfn_range(vma, numgfns, pages); 930 if (rc == 0) 931 free_xenballooned_pages(numpgs, pages); 932 else 933 pr_crit("unable to unmap MFN range: leaking %d pages. rc=%d\n", 934 numpgs, rc); 935 kfree(pages); 936 } 937 938 static vm_fault_t privcmd_fault(struct vm_fault *vmf) 939 { 940 printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n", 941 vmf->vma, vmf->vma->vm_start, vmf->vma->vm_end, 942 vmf->pgoff, (void *)vmf->address); 943 944 return VM_FAULT_SIGBUS; 945 } 946 947 static const struct vm_operations_struct privcmd_vm_ops = { 948 .close = privcmd_close, 949 .fault = privcmd_fault 950 }; 951 952 static int privcmd_mmap(struct file *file, struct vm_area_struct *vma) 953 { 954 /* DONTCOPY is essential for Xen because copy_page_range doesn't know 955 * how to recreate these mappings */ 956 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTCOPY | 957 VM_DONTEXPAND | VM_DONTDUMP; 958 vma->vm_ops = &privcmd_vm_ops; 959 vma->vm_private_data = NULL; 960 961 return 0; 962 } 963 964 /* 965 * For MMAPBATCH*. This allows asserting the singleshot mapping 966 * on a per pfn/pte basis. Mapping calls that fail with ENOENT 967 * can be then retried until success. 968 */ 969 static int is_mapped_fn(pte_t *pte, struct page *pmd_page, 970 unsigned long addr, void *data) 971 { 972 return pte_none(*pte) ? 0 : -EBUSY; 973 } 974 975 static int privcmd_vma_range_is_mapped( 976 struct vm_area_struct *vma, 977 unsigned long addr, 978 unsigned long nr_pages) 979 { 980 return apply_to_page_range(vma->vm_mm, addr, nr_pages << PAGE_SHIFT, 981 is_mapped_fn, NULL) != 0; 982 } 983 984 const struct file_operations xen_privcmd_fops = { 985 .owner = THIS_MODULE, 986 .unlocked_ioctl = privcmd_ioctl, 987 .open = privcmd_open, 988 .release = privcmd_release, 989 .mmap = privcmd_mmap, 990 }; 991 EXPORT_SYMBOL_GPL(xen_privcmd_fops); 992 993 static struct miscdevice privcmd_dev = { 994 .minor = MISC_DYNAMIC_MINOR, 995 .name = "xen/privcmd", 996 .fops = &xen_privcmd_fops, 997 }; 998 999 static int __init privcmd_init(void) 1000 { 1001 int err; 1002 1003 if (!xen_domain()) 1004 return -ENODEV; 1005 1006 err = misc_register(&privcmd_dev); 1007 if (err != 0) { 1008 pr_err("Could not register Xen privcmd device\n"); 1009 return err; 1010 } 1011 1012 err = misc_register(&xen_privcmdbuf_dev); 1013 if (err != 0) { 1014 pr_err("Could not register Xen hypercall-buf device\n"); 1015 misc_deregister(&privcmd_dev); 1016 return err; 1017 } 1018 1019 return 0; 1020 } 1021 1022 static void __exit privcmd_exit(void) 1023 { 1024 misc_deregister(&privcmd_dev); 1025 misc_deregister(&xen_privcmdbuf_dev); 1026 } 1027 1028 module_init(privcmd_init); 1029 module_exit(privcmd_exit); 1030