1 /****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 35 36 #include <linux/module.h> 37 #include <linux/sched.h> 38 #include <linux/mm.h> 39 #include <linux/slab.h> 40 #include <linux/vmalloc.h> 41 #include <linux/uaccess.h> 42 #include <linux/io.h> 43 #include <linux/delay.h> 44 #include <linux/hardirq.h> 45 #include <linux/workqueue.h> 46 47 #include <xen/xen.h> 48 #include <xen/interface/xen.h> 49 #include <xen/page.h> 50 #include <xen/grant_table.h> 51 #include <xen/interface/memory.h> 52 #include <xen/hvc-console.h> 53 #include <xen/swiotlb-xen.h> 54 #include <xen/balloon.h> 55 #include <asm/xen/hypercall.h> 56 #include <asm/xen/interface.h> 57 58 #include <asm/pgtable.h> 59 #include <asm/sync_bitops.h> 60 61 /* External tools reserve first few grant table entries. */ 62 #define NR_RESERVED_ENTRIES 8 63 #define GNTTAB_LIST_END 0xffffffff 64 65 static grant_ref_t **gnttab_list; 66 static unsigned int nr_grant_frames; 67 static int gnttab_free_count; 68 static grant_ref_t gnttab_free_head; 69 static DEFINE_SPINLOCK(gnttab_list_lock); 70 struct grant_frames xen_auto_xlat_grant_frames; 71 72 static union { 73 struct grant_entry_v1 *v1; 74 void *addr; 75 } gnttab_shared; 76 77 /*This is a structure of function pointers for grant table*/ 78 struct gnttab_ops { 79 /* 80 * Mapping a list of frames for storing grant entries. Frames parameter 81 * is used to store grant table address when grant table being setup, 82 * nr_gframes is the number of frames to map grant table. Returning 83 * GNTST_okay means success and negative value means failure. 84 */ 85 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes); 86 /* 87 * Release a list of frames which are mapped in map_frames for grant 88 * entry status. 89 */ 90 void (*unmap_frames)(void); 91 /* 92 * Introducing a valid entry into the grant table, granting the frame of 93 * this grant entry to domain for accessing or transfering. Ref 94 * parameter is reference of this introduced grant entry, domid is id of 95 * granted domain, frame is the page frame to be granted, and flags is 96 * status of the grant entry to be updated. 97 */ 98 void (*update_entry)(grant_ref_t ref, domid_t domid, 99 unsigned long frame, unsigned flags); 100 /* 101 * Stop granting a grant entry to domain for accessing. Ref parameter is 102 * reference of a grant entry whose grant access will be stopped, 103 * readonly is not in use in this function. If the grant entry is 104 * currently mapped for reading or writing, just return failure(==0) 105 * directly and don't tear down the grant access. Otherwise, stop grant 106 * access for this entry and return success(==1). 107 */ 108 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly); 109 /* 110 * Stop granting a grant entry to domain for transfer. Ref parameter is 111 * reference of a grant entry whose grant transfer will be stopped. If 112 * tranfer has not started, just reclaim the grant entry and return 113 * failure(==0). Otherwise, wait for the transfer to complete and then 114 * return the frame. 115 */ 116 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref); 117 /* 118 * Query the status of a grant entry. Ref parameter is reference of 119 * queried grant entry, return value is the status of queried entry. 120 * Detailed status(writing/reading) can be gotten from the return value 121 * by bit operations. 122 */ 123 int (*query_foreign_access)(grant_ref_t ref); 124 }; 125 126 struct unmap_refs_callback_data { 127 struct completion completion; 128 int result; 129 }; 130 131 static struct gnttab_ops *gnttab_interface; 132 133 static int grant_table_version; 134 static int grefs_per_grant_frame; 135 136 static struct gnttab_free_callback *gnttab_free_callback_list; 137 138 static int gnttab_expand(unsigned int req_entries); 139 140 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 141 142 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 143 { 144 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 145 } 146 /* This can be used as an l-value */ 147 #define gnttab_entry(entry) (*__gnttab_entry(entry)) 148 149 static int get_free_entries(unsigned count) 150 { 151 unsigned long flags; 152 int ref, rc = 0; 153 grant_ref_t head; 154 155 spin_lock_irqsave(&gnttab_list_lock, flags); 156 157 if ((gnttab_free_count < count) && 158 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 159 spin_unlock_irqrestore(&gnttab_list_lock, flags); 160 return rc; 161 } 162 163 ref = head = gnttab_free_head; 164 gnttab_free_count -= count; 165 while (count-- > 1) 166 head = gnttab_entry(head); 167 gnttab_free_head = gnttab_entry(head); 168 gnttab_entry(head) = GNTTAB_LIST_END; 169 170 spin_unlock_irqrestore(&gnttab_list_lock, flags); 171 172 return ref; 173 } 174 175 static void do_free_callbacks(void) 176 { 177 struct gnttab_free_callback *callback, *next; 178 179 callback = gnttab_free_callback_list; 180 gnttab_free_callback_list = NULL; 181 182 while (callback != NULL) { 183 next = callback->next; 184 if (gnttab_free_count >= callback->count) { 185 callback->next = NULL; 186 callback->fn(callback->arg); 187 } else { 188 callback->next = gnttab_free_callback_list; 189 gnttab_free_callback_list = callback; 190 } 191 callback = next; 192 } 193 } 194 195 static inline void check_free_callbacks(void) 196 { 197 if (unlikely(gnttab_free_callback_list)) 198 do_free_callbacks(); 199 } 200 201 static void put_free_entry(grant_ref_t ref) 202 { 203 unsigned long flags; 204 spin_lock_irqsave(&gnttab_list_lock, flags); 205 gnttab_entry(ref) = gnttab_free_head; 206 gnttab_free_head = ref; 207 gnttab_free_count++; 208 check_free_callbacks(); 209 spin_unlock_irqrestore(&gnttab_list_lock, flags); 210 } 211 212 /* 213 * Following applies to gnttab_update_entry_v1. 214 * Introducing a valid entry into the grant table: 215 * 1. Write ent->domid. 216 * 2. Write ent->frame: 217 * GTF_permit_access: Frame to which access is permitted. 218 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 219 * frame, or zero if none. 220 * 3. Write memory barrier (WMB). 221 * 4. Write ent->flags, inc. valid type. 222 */ 223 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid, 224 unsigned long frame, unsigned flags) 225 { 226 gnttab_shared.v1[ref].domid = domid; 227 gnttab_shared.v1[ref].frame = frame; 228 wmb(); 229 gnttab_shared.v1[ref].flags = flags; 230 } 231 232 /* 233 * Public grant-issuing interface functions 234 */ 235 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 236 unsigned long frame, int readonly) 237 { 238 gnttab_interface->update_entry(ref, domid, frame, 239 GTF_permit_access | (readonly ? GTF_readonly : 0)); 240 } 241 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 242 243 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 244 int readonly) 245 { 246 int ref; 247 248 ref = get_free_entries(1); 249 if (unlikely(ref < 0)) 250 return -ENOSPC; 251 252 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 253 254 return ref; 255 } 256 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 257 258 static int gnttab_query_foreign_access_v1(grant_ref_t ref) 259 { 260 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing); 261 } 262 263 int gnttab_query_foreign_access(grant_ref_t ref) 264 { 265 return gnttab_interface->query_foreign_access(ref); 266 } 267 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 268 269 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly) 270 { 271 u16 flags, nflags; 272 u16 *pflags; 273 274 pflags = &gnttab_shared.v1[ref].flags; 275 nflags = *pflags; 276 do { 277 flags = nflags; 278 if (flags & (GTF_reading|GTF_writing)) 279 return 0; 280 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags); 281 282 return 1; 283 } 284 285 static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 286 { 287 return gnttab_interface->end_foreign_access_ref(ref, readonly); 288 } 289 290 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 291 { 292 if (_gnttab_end_foreign_access_ref(ref, readonly)) 293 return 1; 294 pr_warn("WARNING: g.e. %#x still in use!\n", ref); 295 return 0; 296 } 297 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 298 299 struct deferred_entry { 300 struct list_head list; 301 grant_ref_t ref; 302 bool ro; 303 uint16_t warn_delay; 304 struct page *page; 305 }; 306 static LIST_HEAD(deferred_list); 307 static void gnttab_handle_deferred(unsigned long); 308 static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0); 309 310 static void gnttab_handle_deferred(unsigned long unused) 311 { 312 unsigned int nr = 10; 313 struct deferred_entry *first = NULL; 314 unsigned long flags; 315 316 spin_lock_irqsave(&gnttab_list_lock, flags); 317 while (nr--) { 318 struct deferred_entry *entry 319 = list_first_entry(&deferred_list, 320 struct deferred_entry, list); 321 322 if (entry == first) 323 break; 324 list_del(&entry->list); 325 spin_unlock_irqrestore(&gnttab_list_lock, flags); 326 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) { 327 put_free_entry(entry->ref); 328 if (entry->page) { 329 pr_debug("freeing g.e. %#x (pfn %#lx)\n", 330 entry->ref, page_to_pfn(entry->page)); 331 __free_page(entry->page); 332 } else 333 pr_info("freeing g.e. %#x\n", entry->ref); 334 kfree(entry); 335 entry = NULL; 336 } else { 337 if (!--entry->warn_delay) 338 pr_info("g.e. %#x still pending\n", entry->ref); 339 if (!first) 340 first = entry; 341 } 342 spin_lock_irqsave(&gnttab_list_lock, flags); 343 if (entry) 344 list_add_tail(&entry->list, &deferred_list); 345 else if (list_empty(&deferred_list)) 346 break; 347 } 348 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) { 349 deferred_timer.expires = jiffies + HZ; 350 add_timer(&deferred_timer); 351 } 352 spin_unlock_irqrestore(&gnttab_list_lock, flags); 353 } 354 355 static void gnttab_add_deferred(grant_ref_t ref, bool readonly, 356 struct page *page) 357 { 358 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC); 359 const char *what = KERN_WARNING "leaking"; 360 361 if (entry) { 362 unsigned long flags; 363 364 entry->ref = ref; 365 entry->ro = readonly; 366 entry->page = page; 367 entry->warn_delay = 60; 368 spin_lock_irqsave(&gnttab_list_lock, flags); 369 list_add_tail(&entry->list, &deferred_list); 370 if (!timer_pending(&deferred_timer)) { 371 deferred_timer.expires = jiffies + HZ; 372 add_timer(&deferred_timer); 373 } 374 spin_unlock_irqrestore(&gnttab_list_lock, flags); 375 what = KERN_DEBUG "deferring"; 376 } 377 printk("%s g.e. %#x (pfn %#lx)\n", 378 what, ref, page ? page_to_pfn(page) : -1); 379 } 380 381 void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 382 unsigned long page) 383 { 384 if (gnttab_end_foreign_access_ref(ref, readonly)) { 385 put_free_entry(ref); 386 if (page != 0) 387 free_page(page); 388 } else 389 gnttab_add_deferred(ref, readonly, 390 page ? virt_to_page(page) : NULL); 391 } 392 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 393 394 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 395 { 396 int ref; 397 398 ref = get_free_entries(1); 399 if (unlikely(ref < 0)) 400 return -ENOSPC; 401 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 402 403 return ref; 404 } 405 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 406 407 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 408 unsigned long pfn) 409 { 410 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer); 411 } 412 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 413 414 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref) 415 { 416 unsigned long frame; 417 u16 flags; 418 u16 *pflags; 419 420 pflags = &gnttab_shared.v1[ref].flags; 421 422 /* 423 * If a transfer is not even yet started, try to reclaim the grant 424 * reference and return failure (== 0). 425 */ 426 while (!((flags = *pflags) & GTF_transfer_committed)) { 427 if (sync_cmpxchg(pflags, flags, 0) == flags) 428 return 0; 429 cpu_relax(); 430 } 431 432 /* If a transfer is in progress then wait until it is completed. */ 433 while (!(flags & GTF_transfer_completed)) { 434 flags = *pflags; 435 cpu_relax(); 436 } 437 438 rmb(); /* Read the frame number /after/ reading completion status. */ 439 frame = gnttab_shared.v1[ref].frame; 440 BUG_ON(frame == 0); 441 442 return frame; 443 } 444 445 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 446 { 447 return gnttab_interface->end_foreign_transfer_ref(ref); 448 } 449 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 450 451 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 452 { 453 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 454 put_free_entry(ref); 455 return frame; 456 } 457 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 458 459 void gnttab_free_grant_reference(grant_ref_t ref) 460 { 461 put_free_entry(ref); 462 } 463 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 464 465 void gnttab_free_grant_references(grant_ref_t head) 466 { 467 grant_ref_t ref; 468 unsigned long flags; 469 int count = 1; 470 if (head == GNTTAB_LIST_END) 471 return; 472 spin_lock_irqsave(&gnttab_list_lock, flags); 473 ref = head; 474 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 475 ref = gnttab_entry(ref); 476 count++; 477 } 478 gnttab_entry(ref) = gnttab_free_head; 479 gnttab_free_head = head; 480 gnttab_free_count += count; 481 check_free_callbacks(); 482 spin_unlock_irqrestore(&gnttab_list_lock, flags); 483 } 484 EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 485 486 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 487 { 488 int h = get_free_entries(count); 489 490 if (h < 0) 491 return -ENOSPC; 492 493 *head = h; 494 495 return 0; 496 } 497 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 498 499 int gnttab_empty_grant_references(const grant_ref_t *private_head) 500 { 501 return (*private_head == GNTTAB_LIST_END); 502 } 503 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 504 505 int gnttab_claim_grant_reference(grant_ref_t *private_head) 506 { 507 grant_ref_t g = *private_head; 508 if (unlikely(g == GNTTAB_LIST_END)) 509 return -ENOSPC; 510 *private_head = gnttab_entry(g); 511 return g; 512 } 513 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 514 515 void gnttab_release_grant_reference(grant_ref_t *private_head, 516 grant_ref_t release) 517 { 518 gnttab_entry(release) = *private_head; 519 *private_head = release; 520 } 521 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 522 523 void gnttab_request_free_callback(struct gnttab_free_callback *callback, 524 void (*fn)(void *), void *arg, u16 count) 525 { 526 unsigned long flags; 527 struct gnttab_free_callback *cb; 528 529 spin_lock_irqsave(&gnttab_list_lock, flags); 530 531 /* Check if the callback is already on the list */ 532 cb = gnttab_free_callback_list; 533 while (cb) { 534 if (cb == callback) 535 goto out; 536 cb = cb->next; 537 } 538 539 callback->fn = fn; 540 callback->arg = arg; 541 callback->count = count; 542 callback->next = gnttab_free_callback_list; 543 gnttab_free_callback_list = callback; 544 check_free_callbacks(); 545 out: 546 spin_unlock_irqrestore(&gnttab_list_lock, flags); 547 } 548 EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 549 550 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 551 { 552 struct gnttab_free_callback **pcb; 553 unsigned long flags; 554 555 spin_lock_irqsave(&gnttab_list_lock, flags); 556 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 557 if (*pcb == callback) { 558 *pcb = callback->next; 559 break; 560 } 561 } 562 spin_unlock_irqrestore(&gnttab_list_lock, flags); 563 } 564 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 565 566 static int grow_gnttab_list(unsigned int more_frames) 567 { 568 unsigned int new_nr_grant_frames, extra_entries, i; 569 unsigned int nr_glist_frames, new_nr_glist_frames; 570 571 BUG_ON(grefs_per_grant_frame == 0); 572 573 new_nr_grant_frames = nr_grant_frames + more_frames; 574 extra_entries = more_frames * grefs_per_grant_frame; 575 576 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 577 new_nr_glist_frames = 578 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 579 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 580 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 581 if (!gnttab_list[i]) 582 goto grow_nomem; 583 } 584 585 586 for (i = grefs_per_grant_frame * nr_grant_frames; 587 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++) 588 gnttab_entry(i) = i + 1; 589 590 gnttab_entry(i) = gnttab_free_head; 591 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames; 592 gnttab_free_count += extra_entries; 593 594 nr_grant_frames = new_nr_grant_frames; 595 596 check_free_callbacks(); 597 598 return 0; 599 600 grow_nomem: 601 while (i-- > nr_glist_frames) 602 free_page((unsigned long) gnttab_list[i]); 603 return -ENOMEM; 604 } 605 606 static unsigned int __max_nr_grant_frames(void) 607 { 608 struct gnttab_query_size query; 609 int rc; 610 611 query.dom = DOMID_SELF; 612 613 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 614 if ((rc < 0) || (query.status != GNTST_okay)) 615 return 4; /* Legacy max supported number of frames */ 616 617 return query.max_nr_frames; 618 } 619 620 unsigned int gnttab_max_grant_frames(void) 621 { 622 unsigned int xen_max = __max_nr_grant_frames(); 623 static unsigned int boot_max_nr_grant_frames; 624 625 /* First time, initialize it properly. */ 626 if (!boot_max_nr_grant_frames) 627 boot_max_nr_grant_frames = __max_nr_grant_frames(); 628 629 if (xen_max > boot_max_nr_grant_frames) 630 return boot_max_nr_grant_frames; 631 return xen_max; 632 } 633 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames); 634 635 int gnttab_setup_auto_xlat_frames(phys_addr_t addr) 636 { 637 xen_pfn_t *pfn; 638 unsigned int max_nr_gframes = __max_nr_grant_frames(); 639 unsigned int i; 640 void *vaddr; 641 642 if (xen_auto_xlat_grant_frames.count) 643 return -EINVAL; 644 645 vaddr = xen_remap(addr, XEN_PAGE_SIZE * max_nr_gframes); 646 if (vaddr == NULL) { 647 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n", 648 &addr); 649 return -ENOMEM; 650 } 651 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL); 652 if (!pfn) { 653 xen_unmap(vaddr); 654 return -ENOMEM; 655 } 656 for (i = 0; i < max_nr_gframes; i++) 657 pfn[i] = XEN_PFN_DOWN(addr) + i; 658 659 xen_auto_xlat_grant_frames.vaddr = vaddr; 660 xen_auto_xlat_grant_frames.pfn = pfn; 661 xen_auto_xlat_grant_frames.count = max_nr_gframes; 662 663 return 0; 664 } 665 EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames); 666 667 void gnttab_free_auto_xlat_frames(void) 668 { 669 if (!xen_auto_xlat_grant_frames.count) 670 return; 671 kfree(xen_auto_xlat_grant_frames.pfn); 672 xen_unmap(xen_auto_xlat_grant_frames.vaddr); 673 674 xen_auto_xlat_grant_frames.pfn = NULL; 675 xen_auto_xlat_grant_frames.count = 0; 676 xen_auto_xlat_grant_frames.vaddr = NULL; 677 } 678 EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); 679 680 /** 681 * gnttab_alloc_pages - alloc pages suitable for grant mapping into 682 * @nr_pages: number of pages to alloc 683 * @pages: returns the pages 684 */ 685 int gnttab_alloc_pages(int nr_pages, struct page **pages) 686 { 687 int i; 688 int ret; 689 690 ret = alloc_xenballooned_pages(nr_pages, pages); 691 if (ret < 0) 692 return ret; 693 694 for (i = 0; i < nr_pages; i++) { 695 #if BITS_PER_LONG < 64 696 struct xen_page_foreign *foreign; 697 698 foreign = kzalloc(sizeof(*foreign), GFP_KERNEL); 699 if (!foreign) { 700 gnttab_free_pages(nr_pages, pages); 701 return -ENOMEM; 702 } 703 set_page_private(pages[i], (unsigned long)foreign); 704 #endif 705 SetPagePrivate(pages[i]); 706 } 707 708 return 0; 709 } 710 EXPORT_SYMBOL(gnttab_alloc_pages); 711 712 /** 713 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages() 714 * @nr_pages; number of pages to free 715 * @pages: the pages 716 */ 717 void gnttab_free_pages(int nr_pages, struct page **pages) 718 { 719 int i; 720 721 for (i = 0; i < nr_pages; i++) { 722 if (PagePrivate(pages[i])) { 723 #if BITS_PER_LONG < 64 724 kfree((void *)page_private(pages[i])); 725 #endif 726 ClearPagePrivate(pages[i]); 727 } 728 } 729 free_xenballooned_pages(nr_pages, pages); 730 } 731 EXPORT_SYMBOL(gnttab_free_pages); 732 733 /* Handling of paged out grant targets (GNTST_eagain) */ 734 #define MAX_DELAY 256 735 static inline void 736 gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status, 737 const char *func) 738 { 739 unsigned delay = 1; 740 741 do { 742 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1)); 743 if (*status == GNTST_eagain) 744 msleep(delay++); 745 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY)); 746 747 if (delay >= MAX_DELAY) { 748 pr_err("%s: %s eagain grant\n", func, current->comm); 749 *status = GNTST_bad_page; 750 } 751 } 752 753 void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count) 754 { 755 struct gnttab_map_grant_ref *op; 756 757 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count)) 758 BUG(); 759 for (op = batch; op < batch + count; op++) 760 if (op->status == GNTST_eagain) 761 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op, 762 &op->status, __func__); 763 } 764 EXPORT_SYMBOL_GPL(gnttab_batch_map); 765 766 void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count) 767 { 768 struct gnttab_copy *op; 769 770 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count)) 771 BUG(); 772 for (op = batch; op < batch + count; op++) 773 if (op->status == GNTST_eagain) 774 gnttab_retry_eagain_gop(GNTTABOP_copy, op, 775 &op->status, __func__); 776 } 777 EXPORT_SYMBOL_GPL(gnttab_batch_copy); 778 779 void gnttab_foreach_grant_in_range(struct page *page, 780 unsigned int offset, 781 unsigned int len, 782 xen_grant_fn_t fn, 783 void *data) 784 { 785 unsigned int goffset; 786 unsigned int glen; 787 unsigned long xen_pfn; 788 789 len = min_t(unsigned int, PAGE_SIZE - offset, len); 790 goffset = xen_offset_in_page(offset); 791 792 xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset); 793 794 while (len) { 795 glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len); 796 fn(pfn_to_gfn(xen_pfn), goffset, glen, data); 797 798 goffset = 0; 799 xen_pfn++; 800 len -= glen; 801 } 802 } 803 EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range); 804 805 void gnttab_foreach_grant(struct page **pages, 806 unsigned int nr_grefs, 807 xen_grant_fn_t fn, 808 void *data) 809 { 810 unsigned int goffset = 0; 811 unsigned long xen_pfn = 0; 812 unsigned int i; 813 814 for (i = 0; i < nr_grefs; i++) { 815 if ((i % XEN_PFN_PER_PAGE) == 0) { 816 xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]); 817 goffset = 0; 818 } 819 820 fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data); 821 822 goffset += XEN_PAGE_SIZE; 823 xen_pfn++; 824 } 825 } 826 827 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, 828 struct gnttab_map_grant_ref *kmap_ops, 829 struct page **pages, unsigned int count) 830 { 831 int i, ret; 832 833 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count); 834 if (ret) 835 return ret; 836 837 for (i = 0; i < count; i++) { 838 /* Retry eagain maps */ 839 if (map_ops[i].status == GNTST_eagain) 840 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, 841 &map_ops[i].status, __func__); 842 843 if (map_ops[i].status == GNTST_okay) { 844 struct xen_page_foreign *foreign; 845 846 SetPageForeign(pages[i]); 847 foreign = xen_page_foreign(pages[i]); 848 foreign->domid = map_ops[i].dom; 849 foreign->gref = map_ops[i].ref; 850 } 851 } 852 853 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count); 854 } 855 EXPORT_SYMBOL_GPL(gnttab_map_refs); 856 857 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, 858 struct gnttab_unmap_grant_ref *kunmap_ops, 859 struct page **pages, unsigned int count) 860 { 861 unsigned int i; 862 int ret; 863 864 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); 865 if (ret) 866 return ret; 867 868 for (i = 0; i < count; i++) 869 ClearPageForeign(pages[i]); 870 871 return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count); 872 } 873 EXPORT_SYMBOL_GPL(gnttab_unmap_refs); 874 875 #define GNTTAB_UNMAP_REFS_DELAY 5 876 877 static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item); 878 879 static void gnttab_unmap_work(struct work_struct *work) 880 { 881 struct gntab_unmap_queue_data 882 *unmap_data = container_of(work, 883 struct gntab_unmap_queue_data, 884 gnttab_work.work); 885 if (unmap_data->age != UINT_MAX) 886 unmap_data->age++; 887 __gnttab_unmap_refs_async(unmap_data); 888 } 889 890 static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) 891 { 892 int ret; 893 int pc; 894 895 for (pc = 0; pc < item->count; pc++) { 896 if (page_count(item->pages[pc]) > 1) { 897 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1); 898 schedule_delayed_work(&item->gnttab_work, 899 msecs_to_jiffies(delay)); 900 return; 901 } 902 } 903 904 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops, 905 item->pages, item->count); 906 item->done(ret, item); 907 } 908 909 void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) 910 { 911 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work); 912 item->age = 0; 913 914 __gnttab_unmap_refs_async(item); 915 } 916 EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async); 917 918 static void unmap_refs_callback(int result, 919 struct gntab_unmap_queue_data *data) 920 { 921 struct unmap_refs_callback_data *d = data->data; 922 923 d->result = result; 924 complete(&d->completion); 925 } 926 927 int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item) 928 { 929 struct unmap_refs_callback_data data; 930 931 init_completion(&data.completion); 932 item->data = &data; 933 item->done = &unmap_refs_callback; 934 gnttab_unmap_refs_async(item); 935 wait_for_completion(&data.completion); 936 937 return data.result; 938 } 939 EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync); 940 941 static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) 942 { 943 int rc; 944 945 rc = arch_gnttab_map_shared(frames, nr_gframes, 946 gnttab_max_grant_frames(), 947 &gnttab_shared.addr); 948 BUG_ON(rc); 949 950 return 0; 951 } 952 953 static void gnttab_unmap_frames_v1(void) 954 { 955 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames); 956 } 957 958 static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 959 { 960 struct gnttab_setup_table setup; 961 xen_pfn_t *frames; 962 unsigned int nr_gframes = end_idx + 1; 963 int rc; 964 965 if (xen_feature(XENFEAT_auto_translated_physmap)) { 966 struct xen_add_to_physmap xatp; 967 unsigned int i = end_idx; 968 rc = 0; 969 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes); 970 /* 971 * Loop backwards, so that the first hypercall has the largest 972 * index, ensuring that the table will grow only once. 973 */ 974 do { 975 xatp.domid = DOMID_SELF; 976 xatp.idx = i; 977 xatp.space = XENMAPSPACE_grant_table; 978 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i]; 979 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp); 980 if (rc != 0) { 981 pr_warn("grant table add_to_physmap failed, err=%d\n", 982 rc); 983 break; 984 } 985 } while (i-- > start_idx); 986 987 return rc; 988 } 989 990 /* No need for kzalloc as it is initialized in following hypercall 991 * GNTTABOP_setup_table. 992 */ 993 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 994 if (!frames) 995 return -ENOMEM; 996 997 setup.dom = DOMID_SELF; 998 setup.nr_frames = nr_gframes; 999 set_xen_guest_handle(setup.frame_list, frames); 1000 1001 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 1002 if (rc == -ENOSYS) { 1003 kfree(frames); 1004 return -ENOSYS; 1005 } 1006 1007 BUG_ON(rc || setup.status); 1008 1009 rc = gnttab_interface->map_frames(frames, nr_gframes); 1010 1011 kfree(frames); 1012 1013 return rc; 1014 } 1015 1016 static struct gnttab_ops gnttab_v1_ops = { 1017 .map_frames = gnttab_map_frames_v1, 1018 .unmap_frames = gnttab_unmap_frames_v1, 1019 .update_entry = gnttab_update_entry_v1, 1020 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1, 1021 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1, 1022 .query_foreign_access = gnttab_query_foreign_access_v1, 1023 }; 1024 1025 static void gnttab_request_version(void) 1026 { 1027 /* Only version 1 is used, which will always be available. */ 1028 grant_table_version = 1; 1029 grefs_per_grant_frame = XEN_PAGE_SIZE / sizeof(struct grant_entry_v1); 1030 gnttab_interface = &gnttab_v1_ops; 1031 1032 pr_info("Grant tables using version %d layout\n", grant_table_version); 1033 } 1034 1035 static int gnttab_setup(void) 1036 { 1037 unsigned int max_nr_gframes; 1038 1039 max_nr_gframes = gnttab_max_grant_frames(); 1040 if (max_nr_gframes < nr_grant_frames) 1041 return -ENOSYS; 1042 1043 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) { 1044 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr; 1045 if (gnttab_shared.addr == NULL) { 1046 pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n", 1047 (unsigned long)xen_auto_xlat_grant_frames.vaddr); 1048 return -ENOMEM; 1049 } 1050 } 1051 return gnttab_map(0, nr_grant_frames - 1); 1052 } 1053 1054 int gnttab_resume(void) 1055 { 1056 gnttab_request_version(); 1057 return gnttab_setup(); 1058 } 1059 1060 int gnttab_suspend(void) 1061 { 1062 if (!xen_feature(XENFEAT_auto_translated_physmap)) 1063 gnttab_interface->unmap_frames(); 1064 return 0; 1065 } 1066 1067 static int gnttab_expand(unsigned int req_entries) 1068 { 1069 int rc; 1070 unsigned int cur, extra; 1071 1072 BUG_ON(grefs_per_grant_frame == 0); 1073 cur = nr_grant_frames; 1074 extra = ((req_entries + (grefs_per_grant_frame-1)) / 1075 grefs_per_grant_frame); 1076 if (cur + extra > gnttab_max_grant_frames()) 1077 return -ENOSPC; 1078 1079 rc = gnttab_map(cur, cur + extra - 1); 1080 if (rc == 0) 1081 rc = grow_gnttab_list(extra); 1082 1083 return rc; 1084 } 1085 1086 int gnttab_init(void) 1087 { 1088 int i; 1089 unsigned long max_nr_grant_frames; 1090 unsigned int max_nr_glist_frames, nr_glist_frames; 1091 unsigned int nr_init_grefs; 1092 int ret; 1093 1094 gnttab_request_version(); 1095 max_nr_grant_frames = gnttab_max_grant_frames(); 1096 nr_grant_frames = 1; 1097 1098 /* Determine the maximum number of frames required for the 1099 * grant reference free list on the current hypervisor. 1100 */ 1101 BUG_ON(grefs_per_grant_frame == 0); 1102 max_nr_glist_frames = (max_nr_grant_frames * 1103 grefs_per_grant_frame / RPP); 1104 1105 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 1106 GFP_KERNEL); 1107 if (gnttab_list == NULL) 1108 return -ENOMEM; 1109 1110 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 1111 for (i = 0; i < nr_glist_frames; i++) { 1112 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 1113 if (gnttab_list[i] == NULL) { 1114 ret = -ENOMEM; 1115 goto ini_nomem; 1116 } 1117 } 1118 1119 ret = arch_gnttab_init(max_nr_grant_frames); 1120 if (ret < 0) 1121 goto ini_nomem; 1122 1123 if (gnttab_setup() < 0) { 1124 ret = -ENODEV; 1125 goto ini_nomem; 1126 } 1127 1128 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame; 1129 1130 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 1131 gnttab_entry(i) = i + 1; 1132 1133 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 1134 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 1135 gnttab_free_head = NR_RESERVED_ENTRIES; 1136 1137 printk("Grant table initialized\n"); 1138 return 0; 1139 1140 ini_nomem: 1141 for (i--; i >= 0; i--) 1142 free_page((unsigned long)gnttab_list[i]); 1143 kfree(gnttab_list); 1144 return ret; 1145 } 1146 EXPORT_SYMBOL_GPL(gnttab_init); 1147 1148 static int __gnttab_init(void) 1149 { 1150 /* Delay grant-table initialization in the PV on HVM case */ 1151 if (xen_hvm_domain()) 1152 return 0; 1153 1154 if (!xen_pv_domain()) 1155 return -ENODEV; 1156 1157 return gnttab_init(); 1158 } 1159 /* Starts after core_initcall so that xen_pvh_gnttab_setup can be called 1160 * beforehand to initialize xen_auto_xlat_grant_frames. */ 1161 core_initcall_sync(__gnttab_init); 1162