1 // SPDX-License-Identifier: GPL-2.0-only 2 /****************************************************************************** 3 * privcmd.c 4 * 5 * Interface to privileged domain-0 commands. 6 * 7 * Copyright (c) 2002-2004, K A Fraser, B Dragovic 8 */ 9 10 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 #include <linux/sched.h> 15 #include <linux/slab.h> 16 #include <linux/string.h> 17 #include <linux/errno.h> 18 #include <linux/mm.h> 19 #include <linux/mman.h> 20 #include <linux/uaccess.h> 21 #include <linux/swap.h> 22 #include <linux/highmem.h> 23 #include <linux/pagemap.h> 24 #include <linux/seq_file.h> 25 #include <linux/miscdevice.h> 26 #include <linux/moduleparam.h> 27 28 #include <asm/xen/hypervisor.h> 29 #include <asm/xen/hypercall.h> 30 31 #include <xen/xen.h> 32 #include <xen/privcmd.h> 33 #include <xen/interface/xen.h> 34 #include <xen/interface/memory.h> 35 #include <xen/interface/hvm/dm_op.h> 36 #include <xen/features.h> 37 #include <xen/page.h> 38 #include <xen/xen-ops.h> 39 #include <xen/balloon.h> 40 41 #include "privcmd.h" 42 43 MODULE_LICENSE("GPL"); 44 45 #define PRIV_VMA_LOCKED ((void *)1) 46 47 static unsigned int privcmd_dm_op_max_num = 16; 48 module_param_named(dm_op_max_nr_bufs, privcmd_dm_op_max_num, uint, 0644); 49 MODULE_PARM_DESC(dm_op_max_nr_bufs, 50 "Maximum number of buffers per dm_op hypercall"); 51 52 static unsigned int privcmd_dm_op_buf_max_size = 4096; 53 module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint, 54 0644); 55 MODULE_PARM_DESC(dm_op_buf_max_size, 56 "Maximum size of a dm_op hypercall buffer"); 57 58 struct privcmd_data { 59 domid_t domid; 60 }; 61 62 static int privcmd_vma_range_is_mapped( 63 struct vm_area_struct *vma, 64 unsigned long addr, 65 unsigned long nr_pages); 66 67 static long privcmd_ioctl_hypercall(struct file *file, void __user *udata) 68 { 69 struct privcmd_data *data = file->private_data; 70 struct privcmd_hypercall hypercall; 71 long ret; 72 73 /* Disallow arbitrary hypercalls if restricted */ 74 if (data->domid != DOMID_INVALID) 75 return -EPERM; 76 77 if (copy_from_user(&hypercall, udata, sizeof(hypercall))) 78 return -EFAULT; 79 80 xen_preemptible_hcall_begin(); 81 ret = privcmd_call(hypercall.op, 82 hypercall.arg[0], hypercall.arg[1], 83 hypercall.arg[2], hypercall.arg[3], 84 hypercall.arg[4]); 85 xen_preemptible_hcall_end(); 86 87 return ret; 88 } 89 90 static void free_page_list(struct list_head *pages) 91 { 92 struct page *p, *n; 93 94 list_for_each_entry_safe(p, n, pages, lru) 95 __free_page(p); 96 97 INIT_LIST_HEAD(pages); 98 } 99 100 /* 101 * Given an array of items in userspace, return a list of pages 102 * containing the data. If copying fails, either because of memory 103 * allocation failure or a problem reading user memory, return an 104 * error code; its up to the caller to dispose of any partial list. 105 */ 106 static int gather_array(struct list_head *pagelist, 107 unsigned nelem, size_t size, 108 const void __user *data) 109 { 110 unsigned pageidx; 111 void *pagedata; 112 int ret; 113 114 if (size > PAGE_SIZE) 115 return 0; 116 117 pageidx = PAGE_SIZE; 118 pagedata = NULL; /* quiet, gcc */ 119 while (nelem--) { 120 if (pageidx > PAGE_SIZE-size) { 121 struct page *page = alloc_page(GFP_KERNEL); 122 123 ret = -ENOMEM; 124 if (page == NULL) 125 goto fail; 126 127 pagedata = page_address(page); 128 129 list_add_tail(&page->lru, pagelist); 130 pageidx = 0; 131 } 132 133 ret = -EFAULT; 134 if (copy_from_user(pagedata + pageidx, data, size)) 135 goto fail; 136 137 data += size; 138 pageidx += size; 139 } 140 141 ret = 0; 142 143 fail: 144 return ret; 145 } 146 147 /* 148 * Call function "fn" on each element of the array fragmented 149 * over a list of pages. 150 */ 151 static int traverse_pages(unsigned nelem, size_t size, 152 struct list_head *pos, 153 int (*fn)(void *data, void *state), 154 void *state) 155 { 156 void *pagedata; 157 unsigned pageidx; 158 int ret = 0; 159 160 BUG_ON(size > PAGE_SIZE); 161 162 pageidx = PAGE_SIZE; 163 pagedata = NULL; /* hush, gcc */ 164 165 while (nelem--) { 166 if (pageidx > PAGE_SIZE-size) { 167 struct page *page; 168 pos = pos->next; 169 page = list_entry(pos, struct page, lru); 170 pagedata = page_address(page); 171 pageidx = 0; 172 } 173 174 ret = (*fn)(pagedata + pageidx, state); 175 if (ret) 176 break; 177 pageidx += size; 178 } 179 180 return ret; 181 } 182 183 /* 184 * Similar to traverse_pages, but use each page as a "block" of 185 * data to be processed as one unit. 186 */ 187 static int traverse_pages_block(unsigned nelem, size_t size, 188 struct list_head *pos, 189 int (*fn)(void *data, int nr, void *state), 190 void *state) 191 { 192 void *pagedata; 193 int ret = 0; 194 195 BUG_ON(size > PAGE_SIZE); 196 197 while (nelem) { 198 int nr = (PAGE_SIZE/size); 199 struct page *page; 200 if (nr > nelem) 201 nr = nelem; 202 pos = pos->next; 203 page = list_entry(pos, struct page, lru); 204 pagedata = page_address(page); 205 ret = (*fn)(pagedata, nr, state); 206 if (ret) 207 break; 208 nelem -= nr; 209 } 210 211 return ret; 212 } 213 214 struct mmap_gfn_state { 215 unsigned long va; 216 struct vm_area_struct *vma; 217 domid_t domain; 218 }; 219 220 static int mmap_gfn_range(void *data, void *state) 221 { 222 struct privcmd_mmap_entry *msg = data; 223 struct mmap_gfn_state *st = state; 224 struct vm_area_struct *vma = st->vma; 225 int rc; 226 227 /* Do not allow range to wrap the address space. */ 228 if ((msg->npages > (LONG_MAX >> PAGE_SHIFT)) || 229 ((unsigned long)(msg->npages << PAGE_SHIFT) >= -st->va)) 230 return -EINVAL; 231 232 /* Range chunks must be contiguous in va space. */ 233 if ((msg->va != st->va) || 234 ((msg->va+(msg->npages<<PAGE_SHIFT)) > vma->vm_end)) 235 return -EINVAL; 236 237 rc = xen_remap_domain_gfn_range(vma, 238 msg->va & PAGE_MASK, 239 msg->mfn, msg->npages, 240 vma->vm_page_prot, 241 st->domain, NULL); 242 if (rc < 0) 243 return rc; 244 245 st->va += msg->npages << PAGE_SHIFT; 246 247 return 0; 248 } 249 250 static long privcmd_ioctl_mmap(struct file *file, void __user *udata) 251 { 252 struct privcmd_data *data = file->private_data; 253 struct privcmd_mmap mmapcmd; 254 struct mm_struct *mm = current->mm; 255 struct vm_area_struct *vma; 256 int rc; 257 LIST_HEAD(pagelist); 258 struct mmap_gfn_state state; 259 260 /* We only support privcmd_ioctl_mmap_batch for non-auto-translated. */ 261 if (xen_feature(XENFEAT_auto_translated_physmap)) 262 return -ENOSYS; 263 264 if (copy_from_user(&mmapcmd, udata, sizeof(mmapcmd))) 265 return -EFAULT; 266 267 /* If restriction is in place, check the domid matches */ 268 if (data->domid != DOMID_INVALID && data->domid != mmapcmd.dom) 269 return -EPERM; 270 271 rc = gather_array(&pagelist, 272 mmapcmd.num, sizeof(struct privcmd_mmap_entry), 273 mmapcmd.entry); 274 275 if (rc || list_empty(&pagelist)) 276 goto out; 277 278 mmap_write_lock(mm); 279 280 { 281 struct page *page = list_first_entry(&pagelist, 282 struct page, lru); 283 struct privcmd_mmap_entry *msg = page_address(page); 284 285 vma = find_vma(mm, msg->va); 286 rc = -EINVAL; 287 288 if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) 289 goto out_up; 290 vma->vm_private_data = PRIV_VMA_LOCKED; 291 } 292 293 state.va = vma->vm_start; 294 state.vma = vma; 295 state.domain = mmapcmd.dom; 296 297 rc = traverse_pages(mmapcmd.num, sizeof(struct privcmd_mmap_entry), 298 &pagelist, 299 mmap_gfn_range, &state); 300 301 302 out_up: 303 mmap_write_unlock(mm); 304 305 out: 306 free_page_list(&pagelist); 307 308 return rc; 309 } 310 311 struct mmap_batch_state { 312 domid_t domain; 313 unsigned long va; 314 struct vm_area_struct *vma; 315 int index; 316 /* A tristate: 317 * 0 for no errors 318 * 1 if at least one error has happened (and no 319 * -ENOENT errors have happened) 320 * -ENOENT if at least 1 -ENOENT has happened. 321 */ 322 int global_error; 323 int version; 324 325 /* User-space gfn array to store errors in the second pass for V1. */ 326 xen_pfn_t __user *user_gfn; 327 /* User-space int array to store errors in the second pass for V2. */ 328 int __user *user_err; 329 }; 330 331 /* auto translated dom0 note: if domU being created is PV, then gfn is 332 * mfn(addr on bus). If it's auto xlated, then gfn is pfn (input to HAP). 333 */ 334 static int mmap_batch_fn(void *data, int nr, void *state) 335 { 336 xen_pfn_t *gfnp = data; 337 struct mmap_batch_state *st = state; 338 struct vm_area_struct *vma = st->vma; 339 struct page **pages = vma->vm_private_data; 340 struct page **cur_pages = NULL; 341 int ret; 342 343 if (xen_feature(XENFEAT_auto_translated_physmap)) 344 cur_pages = &pages[st->index]; 345 346 BUG_ON(nr < 0); 347 ret = xen_remap_domain_gfn_array(st->vma, st->va & PAGE_MASK, gfnp, nr, 348 (int *)gfnp, st->vma->vm_page_prot, 349 st->domain, cur_pages); 350 351 /* Adjust the global_error? */ 352 if (ret != nr) { 353 if (ret == -ENOENT) 354 st->global_error = -ENOENT; 355 else { 356 /* Record that at least one error has happened. */ 357 if (st->global_error == 0) 358 st->global_error = 1; 359 } 360 } 361 st->va += XEN_PAGE_SIZE * nr; 362 st->index += nr / XEN_PFN_PER_PAGE; 363 364 return 0; 365 } 366 367 static int mmap_return_error(int err, struct mmap_batch_state *st) 368 { 369 int ret; 370 371 if (st->version == 1) { 372 if (err) { 373 xen_pfn_t gfn; 374 375 ret = get_user(gfn, st->user_gfn); 376 if (ret < 0) 377 return ret; 378 /* 379 * V1 encodes the error codes in the 32bit top 380 * nibble of the gfn (with its known 381 * limitations vis-a-vis 64 bit callers). 382 */ 383 gfn |= (err == -ENOENT) ? 384 PRIVCMD_MMAPBATCH_PAGED_ERROR : 385 PRIVCMD_MMAPBATCH_MFN_ERROR; 386 return __put_user(gfn, st->user_gfn++); 387 } else 388 st->user_gfn++; 389 } else { /* st->version == 2 */ 390 if (err) 391 return __put_user(err, st->user_err++); 392 else 393 st->user_err++; 394 } 395 396 return 0; 397 } 398 399 static int mmap_return_errors(void *data, int nr, void *state) 400 { 401 struct mmap_batch_state *st = state; 402 int *errs = data; 403 int i; 404 int ret; 405 406 for (i = 0; i < nr; i++) { 407 ret = mmap_return_error(errs[i], st); 408 if (ret < 0) 409 return ret; 410 } 411 return 0; 412 } 413 414 /* Allocate pfns that are then mapped with gfns from foreign domid. Update 415 * the vma with the page info to use later. 416 * Returns: 0 if success, otherwise -errno 417 */ 418 static int alloc_empty_pages(struct vm_area_struct *vma, int numpgs) 419 { 420 int rc; 421 struct page **pages; 422 423 pages = kvcalloc(numpgs, sizeof(pages[0]), GFP_KERNEL); 424 if (pages == NULL) 425 return -ENOMEM; 426 427 rc = xen_alloc_unpopulated_pages(numpgs, pages); 428 if (rc != 0) { 429 pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__, 430 numpgs, rc); 431 kvfree(pages); 432 return -ENOMEM; 433 } 434 BUG_ON(vma->vm_private_data != NULL); 435 vma->vm_private_data = pages; 436 437 return 0; 438 } 439 440 static const struct vm_operations_struct privcmd_vm_ops; 441 442 static long privcmd_ioctl_mmap_batch( 443 struct file *file, void __user *udata, int version) 444 { 445 struct privcmd_data *data = file->private_data; 446 int ret; 447 struct privcmd_mmapbatch_v2 m; 448 struct mm_struct *mm = current->mm; 449 struct vm_area_struct *vma; 450 unsigned long nr_pages; 451 LIST_HEAD(pagelist); 452 struct mmap_batch_state state; 453 454 switch (version) { 455 case 1: 456 if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch))) 457 return -EFAULT; 458 /* Returns per-frame error in m.arr. */ 459 m.err = NULL; 460 if (!access_ok(m.arr, m.num * sizeof(*m.arr))) 461 return -EFAULT; 462 break; 463 case 2: 464 if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2))) 465 return -EFAULT; 466 /* Returns per-frame error code in m.err. */ 467 if (!access_ok(m.err, m.num * (sizeof(*m.err)))) 468 return -EFAULT; 469 break; 470 default: 471 return -EINVAL; 472 } 473 474 /* If restriction is in place, check the domid matches */ 475 if (data->domid != DOMID_INVALID && data->domid != m.dom) 476 return -EPERM; 477 478 nr_pages = DIV_ROUND_UP(m.num, XEN_PFN_PER_PAGE); 479 if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT))) 480 return -EINVAL; 481 482 ret = gather_array(&pagelist, m.num, sizeof(xen_pfn_t), m.arr); 483 484 if (ret) 485 goto out; 486 if (list_empty(&pagelist)) { 487 ret = -EINVAL; 488 goto out; 489 } 490 491 if (version == 2) { 492 /* Zero error array now to only copy back actual errors. */ 493 if (clear_user(m.err, sizeof(int) * m.num)) { 494 ret = -EFAULT; 495 goto out; 496 } 497 } 498 499 mmap_write_lock(mm); 500 501 vma = find_vma(mm, m.addr); 502 if (!vma || 503 vma->vm_ops != &privcmd_vm_ops) { 504 ret = -EINVAL; 505 goto out_unlock; 506 } 507 508 /* 509 * Caller must either: 510 * 511 * Map the whole VMA range, which will also allocate all the 512 * pages required for the auto_translated_physmap case. 513 * 514 * Or 515 * 516 * Map unmapped holes left from a previous map attempt (e.g., 517 * because those foreign frames were previously paged out). 518 */ 519 if (vma->vm_private_data == NULL) { 520 if (m.addr != vma->vm_start || 521 m.addr + (nr_pages << PAGE_SHIFT) != vma->vm_end) { 522 ret = -EINVAL; 523 goto out_unlock; 524 } 525 if (xen_feature(XENFEAT_auto_translated_physmap)) { 526 ret = alloc_empty_pages(vma, nr_pages); 527 if (ret < 0) 528 goto out_unlock; 529 } else 530 vma->vm_private_data = PRIV_VMA_LOCKED; 531 } else { 532 if (m.addr < vma->vm_start || 533 m.addr + (nr_pages << PAGE_SHIFT) > vma->vm_end) { 534 ret = -EINVAL; 535 goto out_unlock; 536 } 537 if (privcmd_vma_range_is_mapped(vma, m.addr, nr_pages)) { 538 ret = -EINVAL; 539 goto out_unlock; 540 } 541 } 542 543 state.domain = m.dom; 544 state.vma = vma; 545 state.va = m.addr; 546 state.index = 0; 547 state.global_error = 0; 548 state.version = version; 549 550 BUILD_BUG_ON(((PAGE_SIZE / sizeof(xen_pfn_t)) % XEN_PFN_PER_PAGE) != 0); 551 /* mmap_batch_fn guarantees ret == 0 */ 552 BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t), 553 &pagelist, mmap_batch_fn, &state)); 554 555 mmap_write_unlock(mm); 556 557 if (state.global_error) { 558 /* Write back errors in second pass. */ 559 state.user_gfn = (xen_pfn_t *)m.arr; 560 state.user_err = m.err; 561 ret = traverse_pages_block(m.num, sizeof(xen_pfn_t), 562 &pagelist, mmap_return_errors, &state); 563 } else 564 ret = 0; 565 566 /* If we have not had any EFAULT-like global errors then set the global 567 * error to -ENOENT if necessary. */ 568 if ((ret == 0) && (state.global_error == -ENOENT)) 569 ret = -ENOENT; 570 571 out: 572 free_page_list(&pagelist); 573 return ret; 574 575 out_unlock: 576 mmap_write_unlock(mm); 577 goto out; 578 } 579 580 static int lock_pages( 581 struct privcmd_dm_op_buf kbufs[], unsigned int num, 582 struct page *pages[], unsigned int nr_pages, unsigned int *pinned) 583 { 584 unsigned int i, off = 0; 585 586 for (i = 0; i < num; ) { 587 unsigned int requested; 588 int page_count; 589 590 requested = DIV_ROUND_UP( 591 offset_in_page(kbufs[i].uptr) + kbufs[i].size, 592 PAGE_SIZE) - off; 593 if (requested > nr_pages) 594 return -ENOSPC; 595 596 page_count = pin_user_pages_fast( 597 (unsigned long)kbufs[i].uptr + off * PAGE_SIZE, 598 requested, FOLL_WRITE, pages); 599 if (page_count <= 0) 600 return page_count ? : -EFAULT; 601 602 *pinned += page_count; 603 nr_pages -= page_count; 604 pages += page_count; 605 606 off = (requested == page_count) ? 0 : off + page_count; 607 i += !off; 608 } 609 610 return 0; 611 } 612 613 static void unlock_pages(struct page *pages[], unsigned int nr_pages) 614 { 615 unpin_user_pages_dirty_lock(pages, nr_pages, true); 616 } 617 618 static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) 619 { 620 struct privcmd_data *data = file->private_data; 621 struct privcmd_dm_op kdata; 622 struct privcmd_dm_op_buf *kbufs; 623 unsigned int nr_pages = 0; 624 struct page **pages = NULL; 625 struct xen_dm_op_buf *xbufs = NULL; 626 unsigned int i; 627 long rc; 628 unsigned int pinned = 0; 629 630 if (copy_from_user(&kdata, udata, sizeof(kdata))) 631 return -EFAULT; 632 633 /* If restriction is in place, check the domid matches */ 634 if (data->domid != DOMID_INVALID && data->domid != kdata.dom) 635 return -EPERM; 636 637 if (kdata.num == 0) 638 return 0; 639 640 if (kdata.num > privcmd_dm_op_max_num) 641 return -E2BIG; 642 643 kbufs = kcalloc(kdata.num, sizeof(*kbufs), GFP_KERNEL); 644 if (!kbufs) 645 return -ENOMEM; 646 647 if (copy_from_user(kbufs, kdata.ubufs, 648 sizeof(*kbufs) * kdata.num)) { 649 rc = -EFAULT; 650 goto out; 651 } 652 653 for (i = 0; i < kdata.num; i++) { 654 if (kbufs[i].size > privcmd_dm_op_buf_max_size) { 655 rc = -E2BIG; 656 goto out; 657 } 658 659 if (!access_ok(kbufs[i].uptr, 660 kbufs[i].size)) { 661 rc = -EFAULT; 662 goto out; 663 } 664 665 nr_pages += DIV_ROUND_UP( 666 offset_in_page(kbufs[i].uptr) + kbufs[i].size, 667 PAGE_SIZE); 668 } 669 670 pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); 671 if (!pages) { 672 rc = -ENOMEM; 673 goto out; 674 } 675 676 xbufs = kcalloc(kdata.num, sizeof(*xbufs), GFP_KERNEL); 677 if (!xbufs) { 678 rc = -ENOMEM; 679 goto out; 680 } 681 682 rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned); 683 if (rc < 0) 684 goto out; 685 686 for (i = 0; i < kdata.num; i++) { 687 set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr); 688 xbufs[i].size = kbufs[i].size; 689 } 690 691 xen_preemptible_hcall_begin(); 692 rc = HYPERVISOR_dm_op(kdata.dom, kdata.num, xbufs); 693 xen_preemptible_hcall_end(); 694 695 out: 696 unlock_pages(pages, pinned); 697 kfree(xbufs); 698 kfree(pages); 699 kfree(kbufs); 700 701 return rc; 702 } 703 704 static long privcmd_ioctl_restrict(struct file *file, void __user *udata) 705 { 706 struct privcmd_data *data = file->private_data; 707 domid_t dom; 708 709 if (copy_from_user(&dom, udata, sizeof(dom))) 710 return -EFAULT; 711 712 /* Set restriction to the specified domain, or check it matches */ 713 if (data->domid == DOMID_INVALID) 714 data->domid = dom; 715 else if (data->domid != dom) 716 return -EINVAL; 717 718 return 0; 719 } 720 721 static long privcmd_ioctl_mmap_resource(struct file *file, 722 struct privcmd_mmap_resource __user *udata) 723 { 724 struct privcmd_data *data = file->private_data; 725 struct mm_struct *mm = current->mm; 726 struct vm_area_struct *vma; 727 struct privcmd_mmap_resource kdata; 728 xen_pfn_t *pfns = NULL; 729 struct xen_mem_acquire_resource xdata = { }; 730 int rc; 731 732 if (copy_from_user(&kdata, udata, sizeof(kdata))) 733 return -EFAULT; 734 735 /* If restriction is in place, check the domid matches */ 736 if (data->domid != DOMID_INVALID && data->domid != kdata.dom) 737 return -EPERM; 738 739 /* Both fields must be set or unset */ 740 if (!!kdata.addr != !!kdata.num) 741 return -EINVAL; 742 743 xdata.domid = kdata.dom; 744 xdata.type = kdata.type; 745 xdata.id = kdata.id; 746 747 if (!kdata.addr && !kdata.num) { 748 /* Query the size of the resource. */ 749 rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, &xdata); 750 if (rc) 751 return rc; 752 return __put_user(xdata.nr_frames, &udata->num); 753 } 754 755 mmap_write_lock(mm); 756 757 vma = find_vma(mm, kdata.addr); 758 if (!vma || vma->vm_ops != &privcmd_vm_ops) { 759 rc = -EINVAL; 760 goto out; 761 } 762 763 pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL); 764 if (!pfns) { 765 rc = -ENOMEM; 766 goto out; 767 } 768 769 if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && 770 xen_feature(XENFEAT_auto_translated_physmap)) { 771 unsigned int nr = DIV_ROUND_UP(kdata.num, XEN_PFN_PER_PAGE); 772 struct page **pages; 773 unsigned int i; 774 775 rc = alloc_empty_pages(vma, nr); 776 if (rc < 0) 777 goto out; 778 779 pages = vma->vm_private_data; 780 for (i = 0; i < kdata.num; i++) { 781 xen_pfn_t pfn = 782 page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]); 783 784 pfns[i] = pfn + (i % XEN_PFN_PER_PAGE); 785 } 786 } else 787 vma->vm_private_data = PRIV_VMA_LOCKED; 788 789 xdata.frame = kdata.idx; 790 xdata.nr_frames = kdata.num; 791 set_xen_guest_handle(xdata.frame_list, pfns); 792 793 xen_preemptible_hcall_begin(); 794 rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, &xdata); 795 xen_preemptible_hcall_end(); 796 797 if (rc) 798 goto out; 799 800 if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && 801 xen_feature(XENFEAT_auto_translated_physmap)) { 802 rc = xen_remap_vma_range(vma, kdata.addr, kdata.num << PAGE_SHIFT); 803 } else { 804 unsigned int domid = 805 (xdata.flags & XENMEM_rsrc_acq_caller_owned) ? 806 DOMID_SELF : kdata.dom; 807 int num, *errs = (int *)pfns; 808 809 BUILD_BUG_ON(sizeof(*errs) > sizeof(*pfns)); 810 num = xen_remap_domain_mfn_array(vma, 811 kdata.addr & PAGE_MASK, 812 pfns, kdata.num, errs, 813 vma->vm_page_prot, 814 domid); 815 if (num < 0) 816 rc = num; 817 else if (num != kdata.num) { 818 unsigned int i; 819 820 for (i = 0; i < num; i++) { 821 rc = errs[i]; 822 if (rc < 0) 823 break; 824 } 825 } else 826 rc = 0; 827 } 828 829 out: 830 mmap_write_unlock(mm); 831 kfree(pfns); 832 833 return rc; 834 } 835 836 static long privcmd_ioctl(struct file *file, 837 unsigned int cmd, unsigned long data) 838 { 839 int ret = -ENOTTY; 840 void __user *udata = (void __user *) data; 841 842 switch (cmd) { 843 case IOCTL_PRIVCMD_HYPERCALL: 844 ret = privcmd_ioctl_hypercall(file, udata); 845 break; 846 847 case IOCTL_PRIVCMD_MMAP: 848 ret = privcmd_ioctl_mmap(file, udata); 849 break; 850 851 case IOCTL_PRIVCMD_MMAPBATCH: 852 ret = privcmd_ioctl_mmap_batch(file, udata, 1); 853 break; 854 855 case IOCTL_PRIVCMD_MMAPBATCH_V2: 856 ret = privcmd_ioctl_mmap_batch(file, udata, 2); 857 break; 858 859 case IOCTL_PRIVCMD_DM_OP: 860 ret = privcmd_ioctl_dm_op(file, udata); 861 break; 862 863 case IOCTL_PRIVCMD_RESTRICT: 864 ret = privcmd_ioctl_restrict(file, udata); 865 break; 866 867 case IOCTL_PRIVCMD_MMAP_RESOURCE: 868 ret = privcmd_ioctl_mmap_resource(file, udata); 869 break; 870 871 default: 872 break; 873 } 874 875 return ret; 876 } 877 878 static int privcmd_open(struct inode *ino, struct file *file) 879 { 880 struct privcmd_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 881 882 if (!data) 883 return -ENOMEM; 884 885 /* DOMID_INVALID implies no restriction */ 886 data->domid = DOMID_INVALID; 887 888 file->private_data = data; 889 return 0; 890 } 891 892 static int privcmd_release(struct inode *ino, struct file *file) 893 { 894 struct privcmd_data *data = file->private_data; 895 896 kfree(data); 897 return 0; 898 } 899 900 static void privcmd_close(struct vm_area_struct *vma) 901 { 902 struct page **pages = vma->vm_private_data; 903 int numpgs = vma_pages(vma); 904 int numgfns = (vma->vm_end - vma->vm_start) >> XEN_PAGE_SHIFT; 905 int rc; 906 907 if (!xen_feature(XENFEAT_auto_translated_physmap) || !numpgs || !pages) 908 return; 909 910 rc = xen_unmap_domain_gfn_range(vma, numgfns, pages); 911 if (rc == 0) 912 xen_free_unpopulated_pages(numpgs, pages); 913 else 914 pr_crit("unable to unmap MFN range: leaking %d pages. rc=%d\n", 915 numpgs, rc); 916 kvfree(pages); 917 } 918 919 static vm_fault_t privcmd_fault(struct vm_fault *vmf) 920 { 921 printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n", 922 vmf->vma, vmf->vma->vm_start, vmf->vma->vm_end, 923 vmf->pgoff, (void *)vmf->address); 924 925 return VM_FAULT_SIGBUS; 926 } 927 928 static const struct vm_operations_struct privcmd_vm_ops = { 929 .close = privcmd_close, 930 .fault = privcmd_fault 931 }; 932 933 static int privcmd_mmap(struct file *file, struct vm_area_struct *vma) 934 { 935 /* DONTCOPY is essential for Xen because copy_page_range doesn't know 936 * how to recreate these mappings */ 937 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTCOPY | 938 VM_DONTEXPAND | VM_DONTDUMP; 939 vma->vm_ops = &privcmd_vm_ops; 940 vma->vm_private_data = NULL; 941 942 return 0; 943 } 944 945 /* 946 * For MMAPBATCH*. This allows asserting the singleshot mapping 947 * on a per pfn/pte basis. Mapping calls that fail with ENOENT 948 * can be then retried until success. 949 */ 950 static int is_mapped_fn(pte_t *pte, unsigned long addr, void *data) 951 { 952 return pte_none(*pte) ? 0 : -EBUSY; 953 } 954 955 static int privcmd_vma_range_is_mapped( 956 struct vm_area_struct *vma, 957 unsigned long addr, 958 unsigned long nr_pages) 959 { 960 return apply_to_page_range(vma->vm_mm, addr, nr_pages << PAGE_SHIFT, 961 is_mapped_fn, NULL) != 0; 962 } 963 964 const struct file_operations xen_privcmd_fops = { 965 .owner = THIS_MODULE, 966 .unlocked_ioctl = privcmd_ioctl, 967 .open = privcmd_open, 968 .release = privcmd_release, 969 .mmap = privcmd_mmap, 970 }; 971 EXPORT_SYMBOL_GPL(xen_privcmd_fops); 972 973 static struct miscdevice privcmd_dev = { 974 .minor = MISC_DYNAMIC_MINOR, 975 .name = "xen/privcmd", 976 .fops = &xen_privcmd_fops, 977 }; 978 979 static int __init privcmd_init(void) 980 { 981 int err; 982 983 if (!xen_domain()) 984 return -ENODEV; 985 986 err = misc_register(&privcmd_dev); 987 if (err != 0) { 988 pr_err("Could not register Xen privcmd device\n"); 989 return err; 990 } 991 992 err = misc_register(&xen_privcmdbuf_dev); 993 if (err != 0) { 994 pr_err("Could not register Xen hypercall-buf device\n"); 995 misc_deregister(&privcmd_dev); 996 return err; 997 } 998 999 return 0; 1000 } 1001 1002 static void __exit privcmd_exit(void) 1003 { 1004 misc_deregister(&privcmd_dev); 1005 misc_deregister(&xen_privcmdbuf_dev); 1006 } 1007 1008 module_init(privcmd_init); 1009 module_exit(privcmd_exit); 1010