1 /****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/sched.h> 36 #include <linux/mm.h> 37 #include <linux/slab.h> 38 #include <linux/vmalloc.h> 39 #include <linux/uaccess.h> 40 41 #include <xen/xen.h> 42 #include <xen/interface/xen.h> 43 #include <xen/page.h> 44 #include <xen/grant_table.h> 45 #include <asm/xen/hypercall.h> 46 47 #include <asm/pgtable.h> 48 #include <asm/sync_bitops.h> 49 50 51 /* External tools reserve first few grant table entries. */ 52 #define NR_RESERVED_ENTRIES 8 53 #define GNTTAB_LIST_END 0xffffffff 54 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry)) 55 56 static grant_ref_t **gnttab_list; 57 static unsigned int nr_grant_frames; 58 static unsigned int boot_max_nr_grant_frames; 59 static int gnttab_free_count; 60 static grant_ref_t gnttab_free_head; 61 static DEFINE_SPINLOCK(gnttab_list_lock); 62 63 static struct grant_entry *shared; 64 65 static struct gnttab_free_callback *gnttab_free_callback_list; 66 67 static int gnttab_expand(unsigned int req_entries); 68 69 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 70 71 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 72 { 73 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 74 } 75 /* This can be used as an l-value */ 76 #define gnttab_entry(entry) (*__gnttab_entry(entry)) 77 78 static int get_free_entries(unsigned count) 79 { 80 unsigned long flags; 81 int ref, rc; 82 grant_ref_t head; 83 84 spin_lock_irqsave(&gnttab_list_lock, flags); 85 86 if ((gnttab_free_count < count) && 87 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 88 spin_unlock_irqrestore(&gnttab_list_lock, flags); 89 return rc; 90 } 91 92 ref = head = gnttab_free_head; 93 gnttab_free_count -= count; 94 while (count-- > 1) 95 head = gnttab_entry(head); 96 gnttab_free_head = gnttab_entry(head); 97 gnttab_entry(head) = GNTTAB_LIST_END; 98 99 spin_unlock_irqrestore(&gnttab_list_lock, flags); 100 101 return ref; 102 } 103 104 static void do_free_callbacks(void) 105 { 106 struct gnttab_free_callback *callback, *next; 107 108 callback = gnttab_free_callback_list; 109 gnttab_free_callback_list = NULL; 110 111 while (callback != NULL) { 112 next = callback->next; 113 if (gnttab_free_count >= callback->count) { 114 callback->next = NULL; 115 callback->fn(callback->arg); 116 } else { 117 callback->next = gnttab_free_callback_list; 118 gnttab_free_callback_list = callback; 119 } 120 callback = next; 121 } 122 } 123 124 static inline void check_free_callbacks(void) 125 { 126 if (unlikely(gnttab_free_callback_list)) 127 do_free_callbacks(); 128 } 129 130 static void put_free_entry(grant_ref_t ref) 131 { 132 unsigned long flags; 133 spin_lock_irqsave(&gnttab_list_lock, flags); 134 gnttab_entry(ref) = gnttab_free_head; 135 gnttab_free_head = ref; 136 gnttab_free_count++; 137 check_free_callbacks(); 138 spin_unlock_irqrestore(&gnttab_list_lock, flags); 139 } 140 141 static void update_grant_entry(grant_ref_t ref, domid_t domid, 142 unsigned long frame, unsigned flags) 143 { 144 /* 145 * Introducing a valid entry into the grant table: 146 * 1. Write ent->domid. 147 * 2. Write ent->frame: 148 * GTF_permit_access: Frame to which access is permitted. 149 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 150 * frame, or zero if none. 151 * 3. Write memory barrier (WMB). 152 * 4. Write ent->flags, inc. valid type. 153 */ 154 shared[ref].frame = frame; 155 shared[ref].domid = domid; 156 wmb(); 157 shared[ref].flags = flags; 158 } 159 160 /* 161 * Public grant-issuing interface functions 162 */ 163 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 164 unsigned long frame, int readonly) 165 { 166 update_grant_entry(ref, domid, frame, 167 GTF_permit_access | (readonly ? GTF_readonly : 0)); 168 } 169 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 170 171 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 172 int readonly) 173 { 174 int ref; 175 176 ref = get_free_entries(1); 177 if (unlikely(ref < 0)) 178 return -ENOSPC; 179 180 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 181 182 return ref; 183 } 184 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 185 186 int gnttab_query_foreign_access(grant_ref_t ref) 187 { 188 u16 nflags; 189 190 nflags = shared[ref].flags; 191 192 return (nflags & (GTF_reading|GTF_writing)); 193 } 194 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 195 196 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 197 { 198 u16 flags, nflags; 199 200 nflags = shared[ref].flags; 201 do { 202 flags = nflags; 203 if (flags & (GTF_reading|GTF_writing)) { 204 printk(KERN_ALERT "WARNING: g.e. still in use!\n"); 205 return 0; 206 } 207 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags); 208 209 return 1; 210 } 211 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 212 213 void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 214 unsigned long page) 215 { 216 if (gnttab_end_foreign_access_ref(ref, readonly)) { 217 put_free_entry(ref); 218 if (page != 0) 219 free_page(page); 220 } else { 221 /* XXX This needs to be fixed so that the ref and page are 222 placed on a list to be freed up later. */ 223 printk(KERN_WARNING 224 "WARNING: leaking g.e. and page still in use!\n"); 225 } 226 } 227 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 228 229 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 230 { 231 int ref; 232 233 ref = get_free_entries(1); 234 if (unlikely(ref < 0)) 235 return -ENOSPC; 236 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 237 238 return ref; 239 } 240 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 241 242 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 243 unsigned long pfn) 244 { 245 update_grant_entry(ref, domid, pfn, GTF_accept_transfer); 246 } 247 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 248 249 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 250 { 251 unsigned long frame; 252 u16 flags; 253 254 /* 255 * If a transfer is not even yet started, try to reclaim the grant 256 * reference and return failure (== 0). 257 */ 258 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 259 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags) 260 return 0; 261 cpu_relax(); 262 } 263 264 /* If a transfer is in progress then wait until it is completed. */ 265 while (!(flags & GTF_transfer_completed)) { 266 flags = shared[ref].flags; 267 cpu_relax(); 268 } 269 270 rmb(); /* Read the frame number /after/ reading completion status. */ 271 frame = shared[ref].frame; 272 BUG_ON(frame == 0); 273 274 return frame; 275 } 276 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 277 278 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 279 { 280 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 281 put_free_entry(ref); 282 return frame; 283 } 284 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 285 286 void gnttab_free_grant_reference(grant_ref_t ref) 287 { 288 put_free_entry(ref); 289 } 290 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 291 292 void gnttab_free_grant_references(grant_ref_t head) 293 { 294 grant_ref_t ref; 295 unsigned long flags; 296 int count = 1; 297 if (head == GNTTAB_LIST_END) 298 return; 299 spin_lock_irqsave(&gnttab_list_lock, flags); 300 ref = head; 301 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 302 ref = gnttab_entry(ref); 303 count++; 304 } 305 gnttab_entry(ref) = gnttab_free_head; 306 gnttab_free_head = head; 307 gnttab_free_count += count; 308 check_free_callbacks(); 309 spin_unlock_irqrestore(&gnttab_list_lock, flags); 310 } 311 EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 312 313 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 314 { 315 int h = get_free_entries(count); 316 317 if (h < 0) 318 return -ENOSPC; 319 320 *head = h; 321 322 return 0; 323 } 324 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 325 326 int gnttab_empty_grant_references(const grant_ref_t *private_head) 327 { 328 return (*private_head == GNTTAB_LIST_END); 329 } 330 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 331 332 int gnttab_claim_grant_reference(grant_ref_t *private_head) 333 { 334 grant_ref_t g = *private_head; 335 if (unlikely(g == GNTTAB_LIST_END)) 336 return -ENOSPC; 337 *private_head = gnttab_entry(g); 338 return g; 339 } 340 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 341 342 void gnttab_release_grant_reference(grant_ref_t *private_head, 343 grant_ref_t release) 344 { 345 gnttab_entry(release) = *private_head; 346 *private_head = release; 347 } 348 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 349 350 void gnttab_request_free_callback(struct gnttab_free_callback *callback, 351 void (*fn)(void *), void *arg, u16 count) 352 { 353 unsigned long flags; 354 spin_lock_irqsave(&gnttab_list_lock, flags); 355 if (callback->next) 356 goto out; 357 callback->fn = fn; 358 callback->arg = arg; 359 callback->count = count; 360 callback->next = gnttab_free_callback_list; 361 gnttab_free_callback_list = callback; 362 check_free_callbacks(); 363 out: 364 spin_unlock_irqrestore(&gnttab_list_lock, flags); 365 } 366 EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 367 368 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 369 { 370 struct gnttab_free_callback **pcb; 371 unsigned long flags; 372 373 spin_lock_irqsave(&gnttab_list_lock, flags); 374 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 375 if (*pcb == callback) { 376 *pcb = callback->next; 377 break; 378 } 379 } 380 spin_unlock_irqrestore(&gnttab_list_lock, flags); 381 } 382 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 383 384 static int grow_gnttab_list(unsigned int more_frames) 385 { 386 unsigned int new_nr_grant_frames, extra_entries, i; 387 unsigned int nr_glist_frames, new_nr_glist_frames; 388 389 new_nr_grant_frames = nr_grant_frames + more_frames; 390 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 391 392 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 393 new_nr_glist_frames = 394 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 395 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 396 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 397 if (!gnttab_list[i]) 398 goto grow_nomem; 399 } 400 401 402 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 403 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 404 gnttab_entry(i) = i + 1; 405 406 gnttab_entry(i) = gnttab_free_head; 407 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 408 gnttab_free_count += extra_entries; 409 410 nr_grant_frames = new_nr_grant_frames; 411 412 check_free_callbacks(); 413 414 return 0; 415 416 grow_nomem: 417 for ( ; i >= nr_glist_frames; i--) 418 free_page((unsigned long) gnttab_list[i]); 419 return -ENOMEM; 420 } 421 422 static unsigned int __max_nr_grant_frames(void) 423 { 424 struct gnttab_query_size query; 425 int rc; 426 427 query.dom = DOMID_SELF; 428 429 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 430 if ((rc < 0) || (query.status != GNTST_okay)) 431 return 4; /* Legacy max supported number of frames */ 432 433 return query.max_nr_frames; 434 } 435 436 static inline unsigned int max_nr_grant_frames(void) 437 { 438 unsigned int xen_max = __max_nr_grant_frames(); 439 440 if (xen_max > boot_max_nr_grant_frames) 441 return boot_max_nr_grant_frames; 442 return xen_max; 443 } 444 445 static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 446 { 447 struct gnttab_setup_table setup; 448 unsigned long *frames; 449 unsigned int nr_gframes = end_idx + 1; 450 int rc; 451 452 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 453 if (!frames) 454 return -ENOMEM; 455 456 setup.dom = DOMID_SELF; 457 setup.nr_frames = nr_gframes; 458 set_xen_guest_handle(setup.frame_list, frames); 459 460 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 461 if (rc == -ENOSYS) { 462 kfree(frames); 463 return -ENOSYS; 464 } 465 466 BUG_ON(rc || setup.status); 467 468 rc = arch_gnttab_map_shared(frames, nr_gframes, max_nr_grant_frames(), 469 &shared); 470 BUG_ON(rc); 471 472 kfree(frames); 473 474 return 0; 475 } 476 477 int gnttab_resume(void) 478 { 479 if (max_nr_grant_frames() < nr_grant_frames) 480 return -ENOSYS; 481 return gnttab_map(0, nr_grant_frames - 1); 482 } 483 484 int gnttab_suspend(void) 485 { 486 arch_gnttab_unmap_shared(shared, nr_grant_frames); 487 return 0; 488 } 489 490 static int gnttab_expand(unsigned int req_entries) 491 { 492 int rc; 493 unsigned int cur, extra; 494 495 cur = nr_grant_frames; 496 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 497 GREFS_PER_GRANT_FRAME); 498 if (cur + extra > max_nr_grant_frames()) 499 return -ENOSPC; 500 501 rc = gnttab_map(cur, cur + extra - 1); 502 if (rc == 0) 503 rc = grow_gnttab_list(extra); 504 505 return rc; 506 } 507 508 static int __devinit gnttab_init(void) 509 { 510 int i; 511 unsigned int max_nr_glist_frames, nr_glist_frames; 512 unsigned int nr_init_grefs; 513 514 if (!xen_domain()) 515 return -ENODEV; 516 517 nr_grant_frames = 1; 518 boot_max_nr_grant_frames = __max_nr_grant_frames(); 519 520 /* Determine the maximum number of frames required for the 521 * grant reference free list on the current hypervisor. 522 */ 523 max_nr_glist_frames = (boot_max_nr_grant_frames * 524 GREFS_PER_GRANT_FRAME / RPP); 525 526 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 527 GFP_KERNEL); 528 if (gnttab_list == NULL) 529 return -ENOMEM; 530 531 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 532 for (i = 0; i < nr_glist_frames; i++) { 533 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 534 if (gnttab_list[i] == NULL) 535 goto ini_nomem; 536 } 537 538 if (gnttab_resume() < 0) 539 return -ENODEV; 540 541 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 542 543 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 544 gnttab_entry(i) = i + 1; 545 546 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 547 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 548 gnttab_free_head = NR_RESERVED_ENTRIES; 549 550 printk("Grant table initialized\n"); 551 return 0; 552 553 ini_nomem: 554 for (i--; i >= 0; i--) 555 free_page((unsigned long)gnttab_list[i]); 556 kfree(gnttab_list); 557 return -ENOMEM; 558 } 559 560 core_initcall(gnttab_init); 561