1 /****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/sched.h> 36 #include <linux/mm.h> 37 #include <linux/slab.h> 38 #include <linux/vmalloc.h> 39 #include <linux/uaccess.h> 40 #include <linux/io.h> 41 42 #include <xen/xen.h> 43 #include <xen/interface/xen.h> 44 #include <xen/page.h> 45 #include <xen/grant_table.h> 46 #include <xen/interface/memory.h> 47 #include <asm/xen/hypercall.h> 48 49 #include <asm/pgtable.h> 50 #include <asm/sync_bitops.h> 51 52 53 /* External tools reserve first few grant table entries. */ 54 #define NR_RESERVED_ENTRIES 8 55 #define GNTTAB_LIST_END 0xffffffff 56 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry)) 57 58 static grant_ref_t **gnttab_list; 59 static unsigned int nr_grant_frames; 60 static unsigned int boot_max_nr_grant_frames; 61 static int gnttab_free_count; 62 static grant_ref_t gnttab_free_head; 63 static DEFINE_SPINLOCK(gnttab_list_lock); 64 unsigned long xen_hvm_resume_frames; 65 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames); 66 67 static struct grant_entry *shared; 68 69 static struct gnttab_free_callback *gnttab_free_callback_list; 70 71 static int gnttab_expand(unsigned int req_entries); 72 73 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 74 75 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 76 { 77 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 78 } 79 /* This can be used as an l-value */ 80 #define gnttab_entry(entry) (*__gnttab_entry(entry)) 81 82 static int get_free_entries(unsigned count) 83 { 84 unsigned long flags; 85 int ref, rc; 86 grant_ref_t head; 87 88 spin_lock_irqsave(&gnttab_list_lock, flags); 89 90 if ((gnttab_free_count < count) && 91 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 92 spin_unlock_irqrestore(&gnttab_list_lock, flags); 93 return rc; 94 } 95 96 ref = head = gnttab_free_head; 97 gnttab_free_count -= count; 98 while (count-- > 1) 99 head = gnttab_entry(head); 100 gnttab_free_head = gnttab_entry(head); 101 gnttab_entry(head) = GNTTAB_LIST_END; 102 103 spin_unlock_irqrestore(&gnttab_list_lock, flags); 104 105 return ref; 106 } 107 108 static void do_free_callbacks(void) 109 { 110 struct gnttab_free_callback *callback, *next; 111 112 callback = gnttab_free_callback_list; 113 gnttab_free_callback_list = NULL; 114 115 while (callback != NULL) { 116 next = callback->next; 117 if (gnttab_free_count >= callback->count) { 118 callback->next = NULL; 119 callback->fn(callback->arg); 120 } else { 121 callback->next = gnttab_free_callback_list; 122 gnttab_free_callback_list = callback; 123 } 124 callback = next; 125 } 126 } 127 128 static inline void check_free_callbacks(void) 129 { 130 if (unlikely(gnttab_free_callback_list)) 131 do_free_callbacks(); 132 } 133 134 static void put_free_entry(grant_ref_t ref) 135 { 136 unsigned long flags; 137 spin_lock_irqsave(&gnttab_list_lock, flags); 138 gnttab_entry(ref) = gnttab_free_head; 139 gnttab_free_head = ref; 140 gnttab_free_count++; 141 check_free_callbacks(); 142 spin_unlock_irqrestore(&gnttab_list_lock, flags); 143 } 144 145 static void update_grant_entry(grant_ref_t ref, domid_t domid, 146 unsigned long frame, unsigned flags) 147 { 148 /* 149 * Introducing a valid entry into the grant table: 150 * 1. Write ent->domid. 151 * 2. Write ent->frame: 152 * GTF_permit_access: Frame to which access is permitted. 153 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 154 * frame, or zero if none. 155 * 3. Write memory barrier (WMB). 156 * 4. Write ent->flags, inc. valid type. 157 */ 158 shared[ref].frame = frame; 159 shared[ref].domid = domid; 160 wmb(); 161 shared[ref].flags = flags; 162 } 163 164 /* 165 * Public grant-issuing interface functions 166 */ 167 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 168 unsigned long frame, int readonly) 169 { 170 update_grant_entry(ref, domid, frame, 171 GTF_permit_access | (readonly ? GTF_readonly : 0)); 172 } 173 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 174 175 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 176 int readonly) 177 { 178 int ref; 179 180 ref = get_free_entries(1); 181 if (unlikely(ref < 0)) 182 return -ENOSPC; 183 184 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 185 186 return ref; 187 } 188 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 189 190 int gnttab_query_foreign_access(grant_ref_t ref) 191 { 192 u16 nflags; 193 194 nflags = shared[ref].flags; 195 196 return (nflags & (GTF_reading|GTF_writing)); 197 } 198 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 199 200 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 201 { 202 u16 flags, nflags; 203 204 nflags = shared[ref].flags; 205 do { 206 flags = nflags; 207 if (flags & (GTF_reading|GTF_writing)) { 208 printk(KERN_ALERT "WARNING: g.e. still in use!\n"); 209 return 0; 210 } 211 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags); 212 213 return 1; 214 } 215 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 216 217 void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 218 unsigned long page) 219 { 220 if (gnttab_end_foreign_access_ref(ref, readonly)) { 221 put_free_entry(ref); 222 if (page != 0) 223 free_page(page); 224 } else { 225 /* XXX This needs to be fixed so that the ref and page are 226 placed on a list to be freed up later. */ 227 printk(KERN_WARNING 228 "WARNING: leaking g.e. and page still in use!\n"); 229 } 230 } 231 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 232 233 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 234 { 235 int ref; 236 237 ref = get_free_entries(1); 238 if (unlikely(ref < 0)) 239 return -ENOSPC; 240 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 241 242 return ref; 243 } 244 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 245 246 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 247 unsigned long pfn) 248 { 249 update_grant_entry(ref, domid, pfn, GTF_accept_transfer); 250 } 251 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 252 253 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 254 { 255 unsigned long frame; 256 u16 flags; 257 258 /* 259 * If a transfer is not even yet started, try to reclaim the grant 260 * reference and return failure (== 0). 261 */ 262 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 263 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags) 264 return 0; 265 cpu_relax(); 266 } 267 268 /* If a transfer is in progress then wait until it is completed. */ 269 while (!(flags & GTF_transfer_completed)) { 270 flags = shared[ref].flags; 271 cpu_relax(); 272 } 273 274 rmb(); /* Read the frame number /after/ reading completion status. */ 275 frame = shared[ref].frame; 276 BUG_ON(frame == 0); 277 278 return frame; 279 } 280 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 281 282 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 283 { 284 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 285 put_free_entry(ref); 286 return frame; 287 } 288 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 289 290 void gnttab_free_grant_reference(grant_ref_t ref) 291 { 292 put_free_entry(ref); 293 } 294 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 295 296 void gnttab_free_grant_references(grant_ref_t head) 297 { 298 grant_ref_t ref; 299 unsigned long flags; 300 int count = 1; 301 if (head == GNTTAB_LIST_END) 302 return; 303 spin_lock_irqsave(&gnttab_list_lock, flags); 304 ref = head; 305 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 306 ref = gnttab_entry(ref); 307 count++; 308 } 309 gnttab_entry(ref) = gnttab_free_head; 310 gnttab_free_head = head; 311 gnttab_free_count += count; 312 check_free_callbacks(); 313 spin_unlock_irqrestore(&gnttab_list_lock, flags); 314 } 315 EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 316 317 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 318 { 319 int h = get_free_entries(count); 320 321 if (h < 0) 322 return -ENOSPC; 323 324 *head = h; 325 326 return 0; 327 } 328 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 329 330 int gnttab_empty_grant_references(const grant_ref_t *private_head) 331 { 332 return (*private_head == GNTTAB_LIST_END); 333 } 334 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 335 336 int gnttab_claim_grant_reference(grant_ref_t *private_head) 337 { 338 grant_ref_t g = *private_head; 339 if (unlikely(g == GNTTAB_LIST_END)) 340 return -ENOSPC; 341 *private_head = gnttab_entry(g); 342 return g; 343 } 344 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 345 346 void gnttab_release_grant_reference(grant_ref_t *private_head, 347 grant_ref_t release) 348 { 349 gnttab_entry(release) = *private_head; 350 *private_head = release; 351 } 352 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 353 354 void gnttab_request_free_callback(struct gnttab_free_callback *callback, 355 void (*fn)(void *), void *arg, u16 count) 356 { 357 unsigned long flags; 358 spin_lock_irqsave(&gnttab_list_lock, flags); 359 if (callback->next) 360 goto out; 361 callback->fn = fn; 362 callback->arg = arg; 363 callback->count = count; 364 callback->next = gnttab_free_callback_list; 365 gnttab_free_callback_list = callback; 366 check_free_callbacks(); 367 out: 368 spin_unlock_irqrestore(&gnttab_list_lock, flags); 369 } 370 EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 371 372 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 373 { 374 struct gnttab_free_callback **pcb; 375 unsigned long flags; 376 377 spin_lock_irqsave(&gnttab_list_lock, flags); 378 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 379 if (*pcb == callback) { 380 *pcb = callback->next; 381 break; 382 } 383 } 384 spin_unlock_irqrestore(&gnttab_list_lock, flags); 385 } 386 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 387 388 static int grow_gnttab_list(unsigned int more_frames) 389 { 390 unsigned int new_nr_grant_frames, extra_entries, i; 391 unsigned int nr_glist_frames, new_nr_glist_frames; 392 393 new_nr_grant_frames = nr_grant_frames + more_frames; 394 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 395 396 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 397 new_nr_glist_frames = 398 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 399 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 400 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 401 if (!gnttab_list[i]) 402 goto grow_nomem; 403 } 404 405 406 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 407 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 408 gnttab_entry(i) = i + 1; 409 410 gnttab_entry(i) = gnttab_free_head; 411 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 412 gnttab_free_count += extra_entries; 413 414 nr_grant_frames = new_nr_grant_frames; 415 416 check_free_callbacks(); 417 418 return 0; 419 420 grow_nomem: 421 for ( ; i >= nr_glist_frames; i--) 422 free_page((unsigned long) gnttab_list[i]); 423 return -ENOMEM; 424 } 425 426 static unsigned int __max_nr_grant_frames(void) 427 { 428 struct gnttab_query_size query; 429 int rc; 430 431 query.dom = DOMID_SELF; 432 433 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 434 if ((rc < 0) || (query.status != GNTST_okay)) 435 return 4; /* Legacy max supported number of frames */ 436 437 return query.max_nr_frames; 438 } 439 440 unsigned int gnttab_max_grant_frames(void) 441 { 442 unsigned int xen_max = __max_nr_grant_frames(); 443 444 if (xen_max > boot_max_nr_grant_frames) 445 return boot_max_nr_grant_frames; 446 return xen_max; 447 } 448 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames); 449 450 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, 451 struct page **pages, unsigned int count) 452 { 453 int i, ret; 454 pte_t *pte; 455 unsigned long mfn; 456 457 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count); 458 if (ret) 459 return ret; 460 461 for (i = 0; i < count; i++) { 462 /* m2p override only supported for GNTMAP_contains_pte mappings */ 463 if (!(map_ops[i].flags & GNTMAP_contains_pte)) 464 continue; 465 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) + 466 (map_ops[i].host_addr & ~PAGE_MASK)); 467 mfn = pte_mfn(*pte); 468 ret = m2p_add_override(mfn, pages[i]); 469 if (ret) 470 return ret; 471 } 472 473 return ret; 474 } 475 EXPORT_SYMBOL_GPL(gnttab_map_refs); 476 477 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, 478 struct page **pages, unsigned int count) 479 { 480 int i, ret; 481 482 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); 483 if (ret) 484 return ret; 485 486 for (i = 0; i < count; i++) { 487 ret = m2p_remove_override(pages[i]); 488 if (ret) 489 return ret; 490 } 491 492 return ret; 493 } 494 EXPORT_SYMBOL_GPL(gnttab_unmap_refs); 495 496 static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 497 { 498 struct gnttab_setup_table setup; 499 unsigned long *frames; 500 unsigned int nr_gframes = end_idx + 1; 501 int rc; 502 503 if (xen_hvm_domain()) { 504 struct xen_add_to_physmap xatp; 505 unsigned int i = end_idx; 506 rc = 0; 507 /* 508 * Loop backwards, so that the first hypercall has the largest 509 * index, ensuring that the table will grow only once. 510 */ 511 do { 512 xatp.domid = DOMID_SELF; 513 xatp.idx = i; 514 xatp.space = XENMAPSPACE_grant_table; 515 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i; 516 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp); 517 if (rc != 0) { 518 printk(KERN_WARNING 519 "grant table add_to_physmap failed, err=%d\n", rc); 520 break; 521 } 522 } while (i-- > start_idx); 523 524 return rc; 525 } 526 527 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 528 if (!frames) 529 return -ENOMEM; 530 531 setup.dom = DOMID_SELF; 532 setup.nr_frames = nr_gframes; 533 set_xen_guest_handle(setup.frame_list, frames); 534 535 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 536 if (rc == -ENOSYS) { 537 kfree(frames); 538 return -ENOSYS; 539 } 540 541 BUG_ON(rc || setup.status); 542 543 rc = arch_gnttab_map_shared(frames, nr_gframes, gnttab_max_grant_frames(), 544 &shared); 545 BUG_ON(rc); 546 547 kfree(frames); 548 549 return 0; 550 } 551 552 int gnttab_resume(void) 553 { 554 unsigned int max_nr_gframes; 555 556 max_nr_gframes = gnttab_max_grant_frames(); 557 if (max_nr_gframes < nr_grant_frames) 558 return -ENOSYS; 559 560 if (xen_pv_domain()) 561 return gnttab_map(0, nr_grant_frames - 1); 562 563 if (!shared) { 564 shared = ioremap(xen_hvm_resume_frames, PAGE_SIZE * max_nr_gframes); 565 if (shared == NULL) { 566 printk(KERN_WARNING 567 "Failed to ioremap gnttab share frames!"); 568 return -ENOMEM; 569 } 570 } 571 572 gnttab_map(0, nr_grant_frames - 1); 573 574 return 0; 575 } 576 577 int gnttab_suspend(void) 578 { 579 arch_gnttab_unmap_shared(shared, nr_grant_frames); 580 return 0; 581 } 582 583 static int gnttab_expand(unsigned int req_entries) 584 { 585 int rc; 586 unsigned int cur, extra; 587 588 cur = nr_grant_frames; 589 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 590 GREFS_PER_GRANT_FRAME); 591 if (cur + extra > gnttab_max_grant_frames()) 592 return -ENOSPC; 593 594 rc = gnttab_map(cur, cur + extra - 1); 595 if (rc == 0) 596 rc = grow_gnttab_list(extra); 597 598 return rc; 599 } 600 601 int gnttab_init(void) 602 { 603 int i; 604 unsigned int max_nr_glist_frames, nr_glist_frames; 605 unsigned int nr_init_grefs; 606 607 nr_grant_frames = 1; 608 boot_max_nr_grant_frames = __max_nr_grant_frames(); 609 610 /* Determine the maximum number of frames required for the 611 * grant reference free list on the current hypervisor. 612 */ 613 max_nr_glist_frames = (boot_max_nr_grant_frames * 614 GREFS_PER_GRANT_FRAME / RPP); 615 616 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 617 GFP_KERNEL); 618 if (gnttab_list == NULL) 619 return -ENOMEM; 620 621 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 622 for (i = 0; i < nr_glist_frames; i++) { 623 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 624 if (gnttab_list[i] == NULL) 625 goto ini_nomem; 626 } 627 628 if (gnttab_resume() < 0) 629 return -ENODEV; 630 631 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 632 633 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 634 gnttab_entry(i) = i + 1; 635 636 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 637 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 638 gnttab_free_head = NR_RESERVED_ENTRIES; 639 640 printk("Grant table initialized\n"); 641 return 0; 642 643 ini_nomem: 644 for (i--; i >= 0; i--) 645 free_page((unsigned long)gnttab_list[i]); 646 kfree(gnttab_list); 647 return -ENOMEM; 648 } 649 EXPORT_SYMBOL_GPL(gnttab_init); 650 651 static int __devinit __gnttab_init(void) 652 { 653 /* Delay grant-table initialization in the PV on HVM case */ 654 if (xen_hvm_domain()) 655 return 0; 656 657 if (!xen_pv_domain()) 658 return -ENODEV; 659 660 return gnttab_init(); 661 } 662 663 core_initcall(__gnttab_init); 664