1 /****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/sched.h> 36 #include <linux/mm.h> 37 #include <linux/vmalloc.h> 38 #include <linux/uaccess.h> 39 40 #include <xen/xen.h> 41 #include <xen/interface/xen.h> 42 #include <xen/page.h> 43 #include <xen/grant_table.h> 44 #include <asm/xen/hypercall.h> 45 46 #include <asm/pgtable.h> 47 #include <asm/sync_bitops.h> 48 49 50 /* External tools reserve first few grant table entries. */ 51 #define NR_RESERVED_ENTRIES 8 52 #define GNTTAB_LIST_END 0xffffffff 53 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry)) 54 55 static grant_ref_t **gnttab_list; 56 static unsigned int nr_grant_frames; 57 static unsigned int boot_max_nr_grant_frames; 58 static int gnttab_free_count; 59 static grant_ref_t gnttab_free_head; 60 static DEFINE_SPINLOCK(gnttab_list_lock); 61 62 static struct grant_entry *shared; 63 64 static struct gnttab_free_callback *gnttab_free_callback_list; 65 66 static int gnttab_expand(unsigned int req_entries); 67 68 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 69 70 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 71 { 72 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 73 } 74 /* This can be used as an l-value */ 75 #define gnttab_entry(entry) (*__gnttab_entry(entry)) 76 77 static int get_free_entries(unsigned count) 78 { 79 unsigned long flags; 80 int ref, rc; 81 grant_ref_t head; 82 83 spin_lock_irqsave(&gnttab_list_lock, flags); 84 85 if ((gnttab_free_count < count) && 86 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 87 spin_unlock_irqrestore(&gnttab_list_lock, flags); 88 return rc; 89 } 90 91 ref = head = gnttab_free_head; 92 gnttab_free_count -= count; 93 while (count-- > 1) 94 head = gnttab_entry(head); 95 gnttab_free_head = gnttab_entry(head); 96 gnttab_entry(head) = GNTTAB_LIST_END; 97 98 spin_unlock_irqrestore(&gnttab_list_lock, flags); 99 100 return ref; 101 } 102 103 static void do_free_callbacks(void) 104 { 105 struct gnttab_free_callback *callback, *next; 106 107 callback = gnttab_free_callback_list; 108 gnttab_free_callback_list = NULL; 109 110 while (callback != NULL) { 111 next = callback->next; 112 if (gnttab_free_count >= callback->count) { 113 callback->next = NULL; 114 callback->fn(callback->arg); 115 } else { 116 callback->next = gnttab_free_callback_list; 117 gnttab_free_callback_list = callback; 118 } 119 callback = next; 120 } 121 } 122 123 static inline void check_free_callbacks(void) 124 { 125 if (unlikely(gnttab_free_callback_list)) 126 do_free_callbacks(); 127 } 128 129 static void put_free_entry(grant_ref_t ref) 130 { 131 unsigned long flags; 132 spin_lock_irqsave(&gnttab_list_lock, flags); 133 gnttab_entry(ref) = gnttab_free_head; 134 gnttab_free_head = ref; 135 gnttab_free_count++; 136 check_free_callbacks(); 137 spin_unlock_irqrestore(&gnttab_list_lock, flags); 138 } 139 140 static void update_grant_entry(grant_ref_t ref, domid_t domid, 141 unsigned long frame, unsigned flags) 142 { 143 /* 144 * Introducing a valid entry into the grant table: 145 * 1. Write ent->domid. 146 * 2. Write ent->frame: 147 * GTF_permit_access: Frame to which access is permitted. 148 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 149 * frame, or zero if none. 150 * 3. Write memory barrier (WMB). 151 * 4. Write ent->flags, inc. valid type. 152 */ 153 shared[ref].frame = frame; 154 shared[ref].domid = domid; 155 wmb(); 156 shared[ref].flags = flags; 157 } 158 159 /* 160 * Public grant-issuing interface functions 161 */ 162 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 163 unsigned long frame, int readonly) 164 { 165 update_grant_entry(ref, domid, frame, 166 GTF_permit_access | (readonly ? GTF_readonly : 0)); 167 } 168 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 169 170 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 171 int readonly) 172 { 173 int ref; 174 175 ref = get_free_entries(1); 176 if (unlikely(ref < 0)) 177 return -ENOSPC; 178 179 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 180 181 return ref; 182 } 183 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 184 185 int gnttab_query_foreign_access(grant_ref_t ref) 186 { 187 u16 nflags; 188 189 nflags = shared[ref].flags; 190 191 return (nflags & (GTF_reading|GTF_writing)); 192 } 193 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 194 195 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 196 { 197 u16 flags, nflags; 198 199 nflags = shared[ref].flags; 200 do { 201 flags = nflags; 202 if (flags & (GTF_reading|GTF_writing)) { 203 printk(KERN_ALERT "WARNING: g.e. still in use!\n"); 204 return 0; 205 } 206 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags); 207 208 return 1; 209 } 210 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 211 212 void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 213 unsigned long page) 214 { 215 if (gnttab_end_foreign_access_ref(ref, readonly)) { 216 put_free_entry(ref); 217 if (page != 0) 218 free_page(page); 219 } else { 220 /* XXX This needs to be fixed so that the ref and page are 221 placed on a list to be freed up later. */ 222 printk(KERN_WARNING 223 "WARNING: leaking g.e. and page still in use!\n"); 224 } 225 } 226 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 227 228 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 229 { 230 int ref; 231 232 ref = get_free_entries(1); 233 if (unlikely(ref < 0)) 234 return -ENOSPC; 235 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 236 237 return ref; 238 } 239 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 240 241 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 242 unsigned long pfn) 243 { 244 update_grant_entry(ref, domid, pfn, GTF_accept_transfer); 245 } 246 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 247 248 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 249 { 250 unsigned long frame; 251 u16 flags; 252 253 /* 254 * If a transfer is not even yet started, try to reclaim the grant 255 * reference and return failure (== 0). 256 */ 257 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 258 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags) 259 return 0; 260 cpu_relax(); 261 } 262 263 /* If a transfer is in progress then wait until it is completed. */ 264 while (!(flags & GTF_transfer_completed)) { 265 flags = shared[ref].flags; 266 cpu_relax(); 267 } 268 269 rmb(); /* Read the frame number /after/ reading completion status. */ 270 frame = shared[ref].frame; 271 BUG_ON(frame == 0); 272 273 return frame; 274 } 275 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 276 277 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 278 { 279 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 280 put_free_entry(ref); 281 return frame; 282 } 283 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 284 285 void gnttab_free_grant_reference(grant_ref_t ref) 286 { 287 put_free_entry(ref); 288 } 289 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 290 291 void gnttab_free_grant_references(grant_ref_t head) 292 { 293 grant_ref_t ref; 294 unsigned long flags; 295 int count = 1; 296 if (head == GNTTAB_LIST_END) 297 return; 298 spin_lock_irqsave(&gnttab_list_lock, flags); 299 ref = head; 300 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 301 ref = gnttab_entry(ref); 302 count++; 303 } 304 gnttab_entry(ref) = gnttab_free_head; 305 gnttab_free_head = head; 306 gnttab_free_count += count; 307 check_free_callbacks(); 308 spin_unlock_irqrestore(&gnttab_list_lock, flags); 309 } 310 EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 311 312 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 313 { 314 int h = get_free_entries(count); 315 316 if (h < 0) 317 return -ENOSPC; 318 319 *head = h; 320 321 return 0; 322 } 323 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 324 325 int gnttab_empty_grant_references(const grant_ref_t *private_head) 326 { 327 return (*private_head == GNTTAB_LIST_END); 328 } 329 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 330 331 int gnttab_claim_grant_reference(grant_ref_t *private_head) 332 { 333 grant_ref_t g = *private_head; 334 if (unlikely(g == GNTTAB_LIST_END)) 335 return -ENOSPC; 336 *private_head = gnttab_entry(g); 337 return g; 338 } 339 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 340 341 void gnttab_release_grant_reference(grant_ref_t *private_head, 342 grant_ref_t release) 343 { 344 gnttab_entry(release) = *private_head; 345 *private_head = release; 346 } 347 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 348 349 void gnttab_request_free_callback(struct gnttab_free_callback *callback, 350 void (*fn)(void *), void *arg, u16 count) 351 { 352 unsigned long flags; 353 spin_lock_irqsave(&gnttab_list_lock, flags); 354 if (callback->next) 355 goto out; 356 callback->fn = fn; 357 callback->arg = arg; 358 callback->count = count; 359 callback->next = gnttab_free_callback_list; 360 gnttab_free_callback_list = callback; 361 check_free_callbacks(); 362 out: 363 spin_unlock_irqrestore(&gnttab_list_lock, flags); 364 } 365 EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 366 367 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 368 { 369 struct gnttab_free_callback **pcb; 370 unsigned long flags; 371 372 spin_lock_irqsave(&gnttab_list_lock, flags); 373 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 374 if (*pcb == callback) { 375 *pcb = callback->next; 376 break; 377 } 378 } 379 spin_unlock_irqrestore(&gnttab_list_lock, flags); 380 } 381 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 382 383 static int grow_gnttab_list(unsigned int more_frames) 384 { 385 unsigned int new_nr_grant_frames, extra_entries, i; 386 unsigned int nr_glist_frames, new_nr_glist_frames; 387 388 new_nr_grant_frames = nr_grant_frames + more_frames; 389 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 390 391 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 392 new_nr_glist_frames = 393 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 394 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 395 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 396 if (!gnttab_list[i]) 397 goto grow_nomem; 398 } 399 400 401 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 402 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 403 gnttab_entry(i) = i + 1; 404 405 gnttab_entry(i) = gnttab_free_head; 406 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 407 gnttab_free_count += extra_entries; 408 409 nr_grant_frames = new_nr_grant_frames; 410 411 check_free_callbacks(); 412 413 return 0; 414 415 grow_nomem: 416 for ( ; i >= nr_glist_frames; i--) 417 free_page((unsigned long) gnttab_list[i]); 418 return -ENOMEM; 419 } 420 421 static unsigned int __max_nr_grant_frames(void) 422 { 423 struct gnttab_query_size query; 424 int rc; 425 426 query.dom = DOMID_SELF; 427 428 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 429 if ((rc < 0) || (query.status != GNTST_okay)) 430 return 4; /* Legacy max supported number of frames */ 431 432 return query.max_nr_frames; 433 } 434 435 static inline unsigned int max_nr_grant_frames(void) 436 { 437 unsigned int xen_max = __max_nr_grant_frames(); 438 439 if (xen_max > boot_max_nr_grant_frames) 440 return boot_max_nr_grant_frames; 441 return xen_max; 442 } 443 444 static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 445 { 446 struct gnttab_setup_table setup; 447 unsigned long *frames; 448 unsigned int nr_gframes = end_idx + 1; 449 int rc; 450 451 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 452 if (!frames) 453 return -ENOMEM; 454 455 setup.dom = DOMID_SELF; 456 setup.nr_frames = nr_gframes; 457 set_xen_guest_handle(setup.frame_list, frames); 458 459 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 460 if (rc == -ENOSYS) { 461 kfree(frames); 462 return -ENOSYS; 463 } 464 465 BUG_ON(rc || setup.status); 466 467 rc = arch_gnttab_map_shared(frames, nr_gframes, max_nr_grant_frames(), 468 &shared); 469 BUG_ON(rc); 470 471 kfree(frames); 472 473 return 0; 474 } 475 476 int gnttab_resume(void) 477 { 478 if (max_nr_grant_frames() < nr_grant_frames) 479 return -ENOSYS; 480 return gnttab_map(0, nr_grant_frames - 1); 481 } 482 483 int gnttab_suspend(void) 484 { 485 arch_gnttab_unmap_shared(shared, nr_grant_frames); 486 return 0; 487 } 488 489 static int gnttab_expand(unsigned int req_entries) 490 { 491 int rc; 492 unsigned int cur, extra; 493 494 cur = nr_grant_frames; 495 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 496 GREFS_PER_GRANT_FRAME); 497 if (cur + extra > max_nr_grant_frames()) 498 return -ENOSPC; 499 500 rc = gnttab_map(cur, cur + extra - 1); 501 if (rc == 0) 502 rc = grow_gnttab_list(extra); 503 504 return rc; 505 } 506 507 static int __devinit gnttab_init(void) 508 { 509 int i; 510 unsigned int max_nr_glist_frames, nr_glist_frames; 511 unsigned int nr_init_grefs; 512 513 if (!xen_domain()) 514 return -ENODEV; 515 516 nr_grant_frames = 1; 517 boot_max_nr_grant_frames = __max_nr_grant_frames(); 518 519 /* Determine the maximum number of frames required for the 520 * grant reference free list on the current hypervisor. 521 */ 522 max_nr_glist_frames = (boot_max_nr_grant_frames * 523 GREFS_PER_GRANT_FRAME / RPP); 524 525 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 526 GFP_KERNEL); 527 if (gnttab_list == NULL) 528 return -ENOMEM; 529 530 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 531 for (i = 0; i < nr_glist_frames; i++) { 532 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 533 if (gnttab_list[i] == NULL) 534 goto ini_nomem; 535 } 536 537 if (gnttab_resume() < 0) 538 return -ENODEV; 539 540 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 541 542 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 543 gnttab_entry(i) = i + 1; 544 545 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 546 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 547 gnttab_free_head = NR_RESERVED_ENTRIES; 548 549 printk("Grant table initialized\n"); 550 return 0; 551 552 ini_nomem: 553 for (i--; i >= 0; i--) 554 free_page((unsigned long)gnttab_list[i]); 555 kfree(gnttab_list); 556 return -ENOMEM; 557 } 558 559 core_initcall(gnttab_init); 560