1 /****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/sched.h> 36 #include <linux/mm.h> 37 #include <linux/vmalloc.h> 38 #include <linux/uaccess.h> 39 40 #include <xen/interface/xen.h> 41 #include <xen/page.h> 42 #include <xen/grant_table.h> 43 #include <asm/xen/hypercall.h> 44 45 #include <asm/pgtable.h> 46 #include <asm/sync_bitops.h> 47 48 49 /* External tools reserve first few grant table entries. */ 50 #define NR_RESERVED_ENTRIES 8 51 #define GNTTAB_LIST_END 0xffffffff 52 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry)) 53 54 static grant_ref_t **gnttab_list; 55 static unsigned int nr_grant_frames; 56 static unsigned int boot_max_nr_grant_frames; 57 static int gnttab_free_count; 58 static grant_ref_t gnttab_free_head; 59 static DEFINE_SPINLOCK(gnttab_list_lock); 60 61 static struct grant_entry *shared; 62 63 static struct gnttab_free_callback *gnttab_free_callback_list; 64 65 static int gnttab_expand(unsigned int req_entries); 66 67 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 68 69 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 70 { 71 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 72 } 73 /* This can be used as an l-value */ 74 #define gnttab_entry(entry) (*__gnttab_entry(entry)) 75 76 static int get_free_entries(unsigned count) 77 { 78 unsigned long flags; 79 int ref, rc; 80 grant_ref_t head; 81 82 spin_lock_irqsave(&gnttab_list_lock, flags); 83 84 if ((gnttab_free_count < count) && 85 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 86 spin_unlock_irqrestore(&gnttab_list_lock, flags); 87 return rc; 88 } 89 90 ref = head = gnttab_free_head; 91 gnttab_free_count -= count; 92 while (count-- > 1) 93 head = gnttab_entry(head); 94 gnttab_free_head = gnttab_entry(head); 95 gnttab_entry(head) = GNTTAB_LIST_END; 96 97 spin_unlock_irqrestore(&gnttab_list_lock, flags); 98 99 return ref; 100 } 101 102 static void do_free_callbacks(void) 103 { 104 struct gnttab_free_callback *callback, *next; 105 106 callback = gnttab_free_callback_list; 107 gnttab_free_callback_list = NULL; 108 109 while (callback != NULL) { 110 next = callback->next; 111 if (gnttab_free_count >= callback->count) { 112 callback->next = NULL; 113 callback->fn(callback->arg); 114 } else { 115 callback->next = gnttab_free_callback_list; 116 gnttab_free_callback_list = callback; 117 } 118 callback = next; 119 } 120 } 121 122 static inline void check_free_callbacks(void) 123 { 124 if (unlikely(gnttab_free_callback_list)) 125 do_free_callbacks(); 126 } 127 128 static void put_free_entry(grant_ref_t ref) 129 { 130 unsigned long flags; 131 spin_lock_irqsave(&gnttab_list_lock, flags); 132 gnttab_entry(ref) = gnttab_free_head; 133 gnttab_free_head = ref; 134 gnttab_free_count++; 135 check_free_callbacks(); 136 spin_unlock_irqrestore(&gnttab_list_lock, flags); 137 } 138 139 static void update_grant_entry(grant_ref_t ref, domid_t domid, 140 unsigned long frame, unsigned flags) 141 { 142 /* 143 * Introducing a valid entry into the grant table: 144 * 1. Write ent->domid. 145 * 2. Write ent->frame: 146 * GTF_permit_access: Frame to which access is permitted. 147 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 148 * frame, or zero if none. 149 * 3. Write memory barrier (WMB). 150 * 4. Write ent->flags, inc. valid type. 151 */ 152 shared[ref].frame = frame; 153 shared[ref].domid = domid; 154 wmb(); 155 shared[ref].flags = flags; 156 } 157 158 /* 159 * Public grant-issuing interface functions 160 */ 161 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 162 unsigned long frame, int readonly) 163 { 164 update_grant_entry(ref, domid, frame, 165 GTF_permit_access | (readonly ? GTF_readonly : 0)); 166 } 167 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 168 169 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 170 int readonly) 171 { 172 int ref; 173 174 ref = get_free_entries(1); 175 if (unlikely(ref < 0)) 176 return -ENOSPC; 177 178 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 179 180 return ref; 181 } 182 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 183 184 int gnttab_query_foreign_access(grant_ref_t ref) 185 { 186 u16 nflags; 187 188 nflags = shared[ref].flags; 189 190 return (nflags & (GTF_reading|GTF_writing)); 191 } 192 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 193 194 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 195 { 196 u16 flags, nflags; 197 198 nflags = shared[ref].flags; 199 do { 200 flags = nflags; 201 if (flags & (GTF_reading|GTF_writing)) { 202 printk(KERN_ALERT "WARNING: g.e. still in use!\n"); 203 return 0; 204 } 205 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags); 206 207 return 1; 208 } 209 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 210 211 void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 212 unsigned long page) 213 { 214 if (gnttab_end_foreign_access_ref(ref, readonly)) { 215 put_free_entry(ref); 216 if (page != 0) 217 free_page(page); 218 } else { 219 /* XXX This needs to be fixed so that the ref and page are 220 placed on a list to be freed up later. */ 221 printk(KERN_WARNING 222 "WARNING: leaking g.e. and page still in use!\n"); 223 } 224 } 225 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 226 227 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 228 { 229 int ref; 230 231 ref = get_free_entries(1); 232 if (unlikely(ref < 0)) 233 return -ENOSPC; 234 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 235 236 return ref; 237 } 238 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 239 240 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 241 unsigned long pfn) 242 { 243 update_grant_entry(ref, domid, pfn, GTF_accept_transfer); 244 } 245 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 246 247 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 248 { 249 unsigned long frame; 250 u16 flags; 251 252 /* 253 * If a transfer is not even yet started, try to reclaim the grant 254 * reference and return failure (== 0). 255 */ 256 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 257 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags) 258 return 0; 259 cpu_relax(); 260 } 261 262 /* If a transfer is in progress then wait until it is completed. */ 263 while (!(flags & GTF_transfer_completed)) { 264 flags = shared[ref].flags; 265 cpu_relax(); 266 } 267 268 rmb(); /* Read the frame number /after/ reading completion status. */ 269 frame = shared[ref].frame; 270 BUG_ON(frame == 0); 271 272 return frame; 273 } 274 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 275 276 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 277 { 278 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 279 put_free_entry(ref); 280 return frame; 281 } 282 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 283 284 void gnttab_free_grant_reference(grant_ref_t ref) 285 { 286 put_free_entry(ref); 287 } 288 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 289 290 void gnttab_free_grant_references(grant_ref_t head) 291 { 292 grant_ref_t ref; 293 unsigned long flags; 294 int count = 1; 295 if (head == GNTTAB_LIST_END) 296 return; 297 spin_lock_irqsave(&gnttab_list_lock, flags); 298 ref = head; 299 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 300 ref = gnttab_entry(ref); 301 count++; 302 } 303 gnttab_entry(ref) = gnttab_free_head; 304 gnttab_free_head = head; 305 gnttab_free_count += count; 306 check_free_callbacks(); 307 spin_unlock_irqrestore(&gnttab_list_lock, flags); 308 } 309 EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 310 311 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 312 { 313 int h = get_free_entries(count); 314 315 if (h < 0) 316 return -ENOSPC; 317 318 *head = h; 319 320 return 0; 321 } 322 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 323 324 int gnttab_empty_grant_references(const grant_ref_t *private_head) 325 { 326 return (*private_head == GNTTAB_LIST_END); 327 } 328 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 329 330 int gnttab_claim_grant_reference(grant_ref_t *private_head) 331 { 332 grant_ref_t g = *private_head; 333 if (unlikely(g == GNTTAB_LIST_END)) 334 return -ENOSPC; 335 *private_head = gnttab_entry(g); 336 return g; 337 } 338 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 339 340 void gnttab_release_grant_reference(grant_ref_t *private_head, 341 grant_ref_t release) 342 { 343 gnttab_entry(release) = *private_head; 344 *private_head = release; 345 } 346 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 347 348 void gnttab_request_free_callback(struct gnttab_free_callback *callback, 349 void (*fn)(void *), void *arg, u16 count) 350 { 351 unsigned long flags; 352 spin_lock_irqsave(&gnttab_list_lock, flags); 353 if (callback->next) 354 goto out; 355 callback->fn = fn; 356 callback->arg = arg; 357 callback->count = count; 358 callback->next = gnttab_free_callback_list; 359 gnttab_free_callback_list = callback; 360 check_free_callbacks(); 361 out: 362 spin_unlock_irqrestore(&gnttab_list_lock, flags); 363 } 364 EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 365 366 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 367 { 368 struct gnttab_free_callback **pcb; 369 unsigned long flags; 370 371 spin_lock_irqsave(&gnttab_list_lock, flags); 372 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 373 if (*pcb == callback) { 374 *pcb = callback->next; 375 break; 376 } 377 } 378 spin_unlock_irqrestore(&gnttab_list_lock, flags); 379 } 380 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 381 382 static int grow_gnttab_list(unsigned int more_frames) 383 { 384 unsigned int new_nr_grant_frames, extra_entries, i; 385 unsigned int nr_glist_frames, new_nr_glist_frames; 386 387 new_nr_grant_frames = nr_grant_frames + more_frames; 388 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 389 390 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 391 new_nr_glist_frames = 392 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 393 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 394 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 395 if (!gnttab_list[i]) 396 goto grow_nomem; 397 } 398 399 400 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 401 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 402 gnttab_entry(i) = i + 1; 403 404 gnttab_entry(i) = gnttab_free_head; 405 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 406 gnttab_free_count += extra_entries; 407 408 nr_grant_frames = new_nr_grant_frames; 409 410 check_free_callbacks(); 411 412 return 0; 413 414 grow_nomem: 415 for ( ; i >= nr_glist_frames; i--) 416 free_page((unsigned long) gnttab_list[i]); 417 return -ENOMEM; 418 } 419 420 static unsigned int __max_nr_grant_frames(void) 421 { 422 struct gnttab_query_size query; 423 int rc; 424 425 query.dom = DOMID_SELF; 426 427 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 428 if ((rc < 0) || (query.status != GNTST_okay)) 429 return 4; /* Legacy max supported number of frames */ 430 431 return query.max_nr_frames; 432 } 433 434 static inline unsigned int max_nr_grant_frames(void) 435 { 436 unsigned int xen_max = __max_nr_grant_frames(); 437 438 if (xen_max > boot_max_nr_grant_frames) 439 return boot_max_nr_grant_frames; 440 return xen_max; 441 } 442 443 static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 444 { 445 struct gnttab_setup_table setup; 446 unsigned long *frames; 447 unsigned int nr_gframes = end_idx + 1; 448 int rc; 449 450 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 451 if (!frames) 452 return -ENOMEM; 453 454 setup.dom = DOMID_SELF; 455 setup.nr_frames = nr_gframes; 456 set_xen_guest_handle(setup.frame_list, frames); 457 458 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 459 if (rc == -ENOSYS) { 460 kfree(frames); 461 return -ENOSYS; 462 } 463 464 BUG_ON(rc || setup.status); 465 466 rc = arch_gnttab_map_shared(frames, nr_gframes, max_nr_grant_frames(), 467 &shared); 468 BUG_ON(rc); 469 470 kfree(frames); 471 472 return 0; 473 } 474 475 int gnttab_resume(void) 476 { 477 if (max_nr_grant_frames() < nr_grant_frames) 478 return -ENOSYS; 479 return gnttab_map(0, nr_grant_frames - 1); 480 } 481 482 int gnttab_suspend(void) 483 { 484 arch_gnttab_unmap_shared(shared, nr_grant_frames); 485 return 0; 486 } 487 488 static int gnttab_expand(unsigned int req_entries) 489 { 490 int rc; 491 unsigned int cur, extra; 492 493 cur = nr_grant_frames; 494 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 495 GREFS_PER_GRANT_FRAME); 496 if (cur + extra > max_nr_grant_frames()) 497 return -ENOSPC; 498 499 rc = gnttab_map(cur, cur + extra - 1); 500 if (rc == 0) 501 rc = grow_gnttab_list(extra); 502 503 return rc; 504 } 505 506 static int __devinit gnttab_init(void) 507 { 508 int i; 509 unsigned int max_nr_glist_frames, nr_glist_frames; 510 unsigned int nr_init_grefs; 511 512 if (!xen_domain()) 513 return -ENODEV; 514 515 nr_grant_frames = 1; 516 boot_max_nr_grant_frames = __max_nr_grant_frames(); 517 518 /* Determine the maximum number of frames required for the 519 * grant reference free list on the current hypervisor. 520 */ 521 max_nr_glist_frames = (boot_max_nr_grant_frames * 522 GREFS_PER_GRANT_FRAME / RPP); 523 524 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 525 GFP_KERNEL); 526 if (gnttab_list == NULL) 527 return -ENOMEM; 528 529 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP; 530 for (i = 0; i < nr_glist_frames; i++) { 531 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 532 if (gnttab_list[i] == NULL) 533 goto ini_nomem; 534 } 535 536 if (gnttab_resume() < 0) 537 return -ENODEV; 538 539 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 540 541 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 542 gnttab_entry(i) = i + 1; 543 544 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 545 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 546 gnttab_free_head = NR_RESERVED_ENTRIES; 547 548 printk("Grant table initialized\n"); 549 return 0; 550 551 ini_nomem: 552 for (i--; i >= 0; i--) 553 free_page((unsigned long)gnttab_list[i]); 554 kfree(gnttab_list); 555 return -ENOMEM; 556 } 557 558 core_initcall(gnttab_init); 559