1 /************************************************************************** 2 * Copyright (c) 2007, Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, write to the Free Software Foundation, Inc., 15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 16 * 17 **************************************************************************/ 18 #include <drm/drmP.h> 19 #include "psb_drv.h" 20 #include "psb_reg.h" 21 22 /* 23 * Code for the SGX MMU: 24 */ 25 26 /* 27 * clflush on one processor only: 28 * clflush should apparently flush the cache line on all processors in an 29 * SMP system. 30 */ 31 32 /* 33 * kmap atomic: 34 * The usage of the slots must be completely encapsulated within a spinlock, and 35 * no other functions that may be using the locks for other purposed may be 36 * called from within the locked region. 37 * Since the slots are per processor, this will guarantee that we are the only 38 * user. 39 */ 40 41 /* 42 * TODO: Inserting ptes from an interrupt handler: 43 * This may be desirable for some SGX functionality where the GPU can fault in 44 * needed pages. For that, we need to make an atomic insert_pages function, that 45 * may fail. 46 * If it fails, the caller need to insert the page using a workqueue function, 47 * but on average it should be fast. 48 */ 49 50 struct psb_mmu_driver { 51 /* protects driver- and pd structures. Always take in read mode 52 * before taking the page table spinlock. 53 */ 54 struct rw_semaphore sem; 55 56 /* protects page tables, directory tables and pt tables. 57 * and pt structures. 58 */ 59 spinlock_t lock; 60 61 atomic_t needs_tlbflush; 62 63 uint8_t __iomem *register_map; 64 struct psb_mmu_pd *default_pd; 65 /*uint32_t bif_ctrl;*/ 66 int has_clflush; 67 int clflush_add; 68 unsigned long clflush_mask; 69 70 struct drm_psb_private *dev_priv; 71 }; 72 73 struct psb_mmu_pd; 74 75 struct psb_mmu_pt { 76 struct psb_mmu_pd *pd; 77 uint32_t index; 78 uint32_t count; 79 struct page *p; 80 uint32_t *v; 81 }; 82 83 struct psb_mmu_pd { 84 struct psb_mmu_driver *driver; 85 int hw_context; 86 struct psb_mmu_pt **tables; 87 struct page *p; 88 struct page *dummy_pt; 89 struct page *dummy_page; 90 uint32_t pd_mask; 91 uint32_t invalid_pde; 92 uint32_t invalid_pte; 93 }; 94 95 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 96 { 97 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 98 } 99 100 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 101 { 102 return offset >> PSB_PDE_SHIFT; 103 } 104 105 static inline void psb_clflush(void *addr) 106 { 107 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 108 } 109 110 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, 111 void *addr) 112 { 113 if (!driver->has_clflush) 114 return; 115 116 mb(); 117 psb_clflush(addr); 118 mb(); 119 } 120 121 static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page) 122 { 123 uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT; 124 uint32_t clflush_count = PAGE_SIZE / clflush_add; 125 int i; 126 uint8_t *clf; 127 128 clf = kmap_atomic(page); 129 mb(); 130 for (i = 0; i < clflush_count; ++i) { 131 psb_clflush(clf); 132 clf += clflush_add; 133 } 134 mb(); 135 kunmap_atomic(clf); 136 } 137 138 static void psb_pages_clflush(struct psb_mmu_driver *driver, 139 struct page *page[], unsigned long num_pages) 140 { 141 int i; 142 143 if (!driver->has_clflush) 144 return ; 145 146 for (i = 0; i < num_pages; i++) 147 psb_page_clflush(driver, *page++); 148 } 149 150 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, 151 int force) 152 { 153 atomic_set(&driver->needs_tlbflush, 0); 154 } 155 156 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 157 { 158 down_write(&driver->sem); 159 psb_mmu_flush_pd_locked(driver, force); 160 up_write(&driver->sem); 161 } 162 163 void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot) 164 { 165 if (rc_prot) 166 down_write(&driver->sem); 167 if (rc_prot) 168 up_write(&driver->sem); 169 } 170 171 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 172 { 173 /*ttm_tt_cache_flush(&pd->p, 1);*/ 174 psb_pages_clflush(pd->driver, &pd->p, 1); 175 down_write(&pd->driver->sem); 176 wmb(); 177 psb_mmu_flush_pd_locked(pd->driver, 1); 178 pd->hw_context = hw_context; 179 up_write(&pd->driver->sem); 180 181 } 182 183 static inline unsigned long psb_pd_addr_end(unsigned long addr, 184 unsigned long end) 185 { 186 187 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 188 return (addr < end) ? addr : end; 189 } 190 191 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 192 { 193 uint32_t mask = PSB_PTE_VALID; 194 195 if (type & PSB_MMU_CACHED_MEMORY) 196 mask |= PSB_PTE_CACHED; 197 if (type & PSB_MMU_RO_MEMORY) 198 mask |= PSB_PTE_RO; 199 if (type & PSB_MMU_WO_MEMORY) 200 mask |= PSB_PTE_WO; 201 202 return (pfn << PAGE_SHIFT) | mask; 203 } 204 205 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 206 int trap_pagefaults, int invalid_type) 207 { 208 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 209 uint32_t *v; 210 int i; 211 212 if (!pd) 213 return NULL; 214 215 pd->p = alloc_page(GFP_DMA32); 216 if (!pd->p) 217 goto out_err1; 218 pd->dummy_pt = alloc_page(GFP_DMA32); 219 if (!pd->dummy_pt) 220 goto out_err2; 221 pd->dummy_page = alloc_page(GFP_DMA32); 222 if (!pd->dummy_page) 223 goto out_err3; 224 225 if (!trap_pagefaults) { 226 pd->invalid_pde = 227 psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 228 invalid_type); 229 pd->invalid_pte = 230 psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 231 invalid_type); 232 } else { 233 pd->invalid_pde = 0; 234 pd->invalid_pte = 0; 235 } 236 237 v = kmap(pd->dummy_pt); 238 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 239 v[i] = pd->invalid_pte; 240 241 kunmap(pd->dummy_pt); 242 243 v = kmap(pd->p); 244 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 245 v[i] = pd->invalid_pde; 246 247 kunmap(pd->p); 248 249 clear_page(kmap(pd->dummy_page)); 250 kunmap(pd->dummy_page); 251 252 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 253 if (!pd->tables) 254 goto out_err4; 255 256 pd->hw_context = -1; 257 pd->pd_mask = PSB_PTE_VALID; 258 pd->driver = driver; 259 260 return pd; 261 262 out_err4: 263 __free_page(pd->dummy_page); 264 out_err3: 265 __free_page(pd->dummy_pt); 266 out_err2: 267 __free_page(pd->p); 268 out_err1: 269 kfree(pd); 270 return NULL; 271 } 272 273 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 274 { 275 __free_page(pt->p); 276 kfree(pt); 277 } 278 279 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 280 { 281 struct psb_mmu_driver *driver = pd->driver; 282 struct psb_mmu_pt *pt; 283 int i; 284 285 down_write(&driver->sem); 286 if (pd->hw_context != -1) 287 psb_mmu_flush_pd_locked(driver, 1); 288 289 /* Should take the spinlock here, but we don't need to do that 290 since we have the semaphore in write mode. */ 291 292 for (i = 0; i < 1024; ++i) { 293 pt = pd->tables[i]; 294 if (pt) 295 psb_mmu_free_pt(pt); 296 } 297 298 vfree(pd->tables); 299 __free_page(pd->dummy_page); 300 __free_page(pd->dummy_pt); 301 __free_page(pd->p); 302 kfree(pd); 303 up_write(&driver->sem); 304 } 305 306 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 307 { 308 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 309 void *v; 310 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 311 uint32_t clflush_count = PAGE_SIZE / clflush_add; 312 spinlock_t *lock = &pd->driver->lock; 313 uint8_t *clf; 314 uint32_t *ptes; 315 int i; 316 317 if (!pt) 318 return NULL; 319 320 pt->p = alloc_page(GFP_DMA32); 321 if (!pt->p) { 322 kfree(pt); 323 return NULL; 324 } 325 326 spin_lock(lock); 327 328 v = kmap_atomic(pt->p); 329 clf = (uint8_t *) v; 330 ptes = (uint32_t *) v; 331 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 332 *ptes++ = pd->invalid_pte; 333 334 335 if (pd->driver->has_clflush && pd->hw_context != -1) { 336 mb(); 337 for (i = 0; i < clflush_count; ++i) { 338 psb_clflush(clf); 339 clf += clflush_add; 340 } 341 mb(); 342 } 343 344 kunmap_atomic(v); 345 spin_unlock(lock); 346 347 pt->count = 0; 348 pt->pd = pd; 349 pt->index = 0; 350 351 return pt; 352 } 353 354 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 355 unsigned long addr) 356 { 357 uint32_t index = psb_mmu_pd_index(addr); 358 struct psb_mmu_pt *pt; 359 uint32_t *v; 360 spinlock_t *lock = &pd->driver->lock; 361 362 spin_lock(lock); 363 pt = pd->tables[index]; 364 while (!pt) { 365 spin_unlock(lock); 366 pt = psb_mmu_alloc_pt(pd); 367 if (!pt) 368 return NULL; 369 spin_lock(lock); 370 371 if (pd->tables[index]) { 372 spin_unlock(lock); 373 psb_mmu_free_pt(pt); 374 spin_lock(lock); 375 pt = pd->tables[index]; 376 continue; 377 } 378 379 v = kmap_atomic(pd->p); 380 pd->tables[index] = pt; 381 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 382 pt->index = index; 383 kunmap_atomic((void *) v); 384 385 if (pd->hw_context != -1) { 386 psb_mmu_clflush(pd->driver, (void *) &v[index]); 387 atomic_set(&pd->driver->needs_tlbflush, 1); 388 } 389 } 390 pt->v = kmap_atomic(pt->p); 391 return pt; 392 } 393 394 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 395 unsigned long addr) 396 { 397 uint32_t index = psb_mmu_pd_index(addr); 398 struct psb_mmu_pt *pt; 399 spinlock_t *lock = &pd->driver->lock; 400 401 spin_lock(lock); 402 pt = pd->tables[index]; 403 if (!pt) { 404 spin_unlock(lock); 405 return NULL; 406 } 407 pt->v = kmap_atomic(pt->p); 408 return pt; 409 } 410 411 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 412 { 413 struct psb_mmu_pd *pd = pt->pd; 414 uint32_t *v; 415 416 kunmap_atomic(pt->v); 417 if (pt->count == 0) { 418 v = kmap_atomic(pd->p); 419 v[pt->index] = pd->invalid_pde; 420 pd->tables[pt->index] = NULL; 421 422 if (pd->hw_context != -1) { 423 psb_mmu_clflush(pd->driver, 424 (void *) &v[pt->index]); 425 atomic_set(&pd->driver->needs_tlbflush, 1); 426 } 427 kunmap_atomic(pt->v); 428 spin_unlock(&pd->driver->lock); 429 psb_mmu_free_pt(pt); 430 return; 431 } 432 spin_unlock(&pd->driver->lock); 433 } 434 435 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, 436 unsigned long addr, uint32_t pte) 437 { 438 pt->v[psb_mmu_pt_index(addr)] = pte; 439 } 440 441 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 442 unsigned long addr) 443 { 444 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 445 } 446 447 448 void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd, 449 uint32_t mmu_offset, uint32_t gtt_start, 450 uint32_t gtt_pages) 451 { 452 uint32_t *v; 453 uint32_t start = psb_mmu_pd_index(mmu_offset); 454 struct psb_mmu_driver *driver = pd->driver; 455 int num_pages = gtt_pages; 456 457 down_read(&driver->sem); 458 spin_lock(&driver->lock); 459 460 v = kmap_atomic(pd->p); 461 v += start; 462 463 while (gtt_pages--) { 464 *v++ = gtt_start | pd->pd_mask; 465 gtt_start += PAGE_SIZE; 466 } 467 468 /*ttm_tt_cache_flush(&pd->p, num_pages);*/ 469 psb_pages_clflush(pd->driver, &pd->p, num_pages); 470 kunmap_atomic(v); 471 spin_unlock(&driver->lock); 472 473 if (pd->hw_context != -1) 474 atomic_set(&pd->driver->needs_tlbflush, 1); 475 476 up_read(&pd->driver->sem); 477 psb_mmu_flush_pd(pd->driver, 0); 478 } 479 480 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 481 { 482 struct psb_mmu_pd *pd; 483 484 /* down_read(&driver->sem); */ 485 pd = driver->default_pd; 486 /* up_read(&driver->sem); */ 487 488 return pd; 489 } 490 491 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 492 { 493 psb_mmu_free_pagedir(driver->default_pd); 494 kfree(driver); 495 } 496 497 struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, 498 int trap_pagefaults, 499 int invalid_type, 500 struct drm_psb_private *dev_priv) 501 { 502 struct psb_mmu_driver *driver; 503 504 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 505 506 if (!driver) 507 return NULL; 508 driver->dev_priv = dev_priv; 509 510 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 511 invalid_type); 512 if (!driver->default_pd) 513 goto out_err1; 514 515 spin_lock_init(&driver->lock); 516 init_rwsem(&driver->sem); 517 down_write(&driver->sem); 518 driver->register_map = registers; 519 atomic_set(&driver->needs_tlbflush, 1); 520 521 driver->has_clflush = 0; 522 523 if (boot_cpu_has(X86_FEATURE_CLFLSH)) { 524 uint32_t tfms, misc, cap0, cap4, clflush_size; 525 526 /* 527 * clflush size is determined at kernel setup for x86_64 528 * but not for i386. We have to do it here. 529 */ 530 531 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 532 clflush_size = ((misc >> 8) & 0xff) * 8; 533 driver->has_clflush = 1; 534 driver->clflush_add = 535 PAGE_SIZE * clflush_size / sizeof(uint32_t); 536 driver->clflush_mask = driver->clflush_add - 1; 537 driver->clflush_mask = ~driver->clflush_mask; 538 } 539 540 up_write(&driver->sem); 541 return driver; 542 543 out_err1: 544 kfree(driver); 545 return NULL; 546 } 547 548 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, 549 unsigned long address, uint32_t num_pages, 550 uint32_t desired_tile_stride, 551 uint32_t hw_tile_stride) 552 { 553 struct psb_mmu_pt *pt; 554 uint32_t rows = 1; 555 uint32_t i; 556 unsigned long addr; 557 unsigned long end; 558 unsigned long next; 559 unsigned long add; 560 unsigned long row_add; 561 unsigned long clflush_add = pd->driver->clflush_add; 562 unsigned long clflush_mask = pd->driver->clflush_mask; 563 564 if (!pd->driver->has_clflush) { 565 /*ttm_tt_cache_flush(&pd->p, num_pages);*/ 566 psb_pages_clflush(pd->driver, &pd->p, num_pages); 567 return; 568 } 569 570 if (hw_tile_stride) 571 rows = num_pages / desired_tile_stride; 572 else 573 desired_tile_stride = num_pages; 574 575 add = desired_tile_stride << PAGE_SHIFT; 576 row_add = hw_tile_stride << PAGE_SHIFT; 577 mb(); 578 for (i = 0; i < rows; ++i) { 579 580 addr = address; 581 end = addr + add; 582 583 do { 584 next = psb_pd_addr_end(addr, end); 585 pt = psb_mmu_pt_map_lock(pd, addr); 586 if (!pt) 587 continue; 588 do { 589 psb_clflush(&pt->v 590 [psb_mmu_pt_index(addr)]); 591 } while (addr += 592 clflush_add, 593 (addr & clflush_mask) < next); 594 595 psb_mmu_pt_unmap_unlock(pt); 596 } while (addr = next, next != end); 597 address += row_add; 598 } 599 mb(); 600 } 601 602 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 603 unsigned long address, uint32_t num_pages) 604 { 605 struct psb_mmu_pt *pt; 606 unsigned long addr; 607 unsigned long end; 608 unsigned long next; 609 unsigned long f_address = address; 610 611 down_read(&pd->driver->sem); 612 613 addr = address; 614 end = addr + (num_pages << PAGE_SHIFT); 615 616 do { 617 next = psb_pd_addr_end(addr, end); 618 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 619 if (!pt) 620 goto out; 621 do { 622 psb_mmu_invalidate_pte(pt, addr); 623 --pt->count; 624 } while (addr += PAGE_SIZE, addr < next); 625 psb_mmu_pt_unmap_unlock(pt); 626 627 } while (addr = next, next != end); 628 629 out: 630 if (pd->hw_context != -1) 631 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 632 633 up_read(&pd->driver->sem); 634 635 if (pd->hw_context != -1) 636 psb_mmu_flush(pd->driver, 0); 637 638 return; 639 } 640 641 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 642 uint32_t num_pages, uint32_t desired_tile_stride, 643 uint32_t hw_tile_stride) 644 { 645 struct psb_mmu_pt *pt; 646 uint32_t rows = 1; 647 uint32_t i; 648 unsigned long addr; 649 unsigned long end; 650 unsigned long next; 651 unsigned long add; 652 unsigned long row_add; 653 unsigned long f_address = address; 654 655 if (hw_tile_stride) 656 rows = num_pages / desired_tile_stride; 657 else 658 desired_tile_stride = num_pages; 659 660 add = desired_tile_stride << PAGE_SHIFT; 661 row_add = hw_tile_stride << PAGE_SHIFT; 662 663 /* down_read(&pd->driver->sem); */ 664 665 /* Make sure we only need to flush this processor's cache */ 666 667 for (i = 0; i < rows; ++i) { 668 669 addr = address; 670 end = addr + add; 671 672 do { 673 next = psb_pd_addr_end(addr, end); 674 pt = psb_mmu_pt_map_lock(pd, addr); 675 if (!pt) 676 continue; 677 do { 678 psb_mmu_invalidate_pte(pt, addr); 679 --pt->count; 680 681 } while (addr += PAGE_SIZE, addr < next); 682 psb_mmu_pt_unmap_unlock(pt); 683 684 } while (addr = next, next != end); 685 address += row_add; 686 } 687 if (pd->hw_context != -1) 688 psb_mmu_flush_ptes(pd, f_address, num_pages, 689 desired_tile_stride, hw_tile_stride); 690 691 /* up_read(&pd->driver->sem); */ 692 693 if (pd->hw_context != -1) 694 psb_mmu_flush(pd->driver, 0); 695 } 696 697 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 698 unsigned long address, uint32_t num_pages, 699 int type) 700 { 701 struct psb_mmu_pt *pt; 702 uint32_t pte; 703 unsigned long addr; 704 unsigned long end; 705 unsigned long next; 706 unsigned long f_address = address; 707 int ret = 0; 708 709 down_read(&pd->driver->sem); 710 711 addr = address; 712 end = addr + (num_pages << PAGE_SHIFT); 713 714 do { 715 next = psb_pd_addr_end(addr, end); 716 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 717 if (!pt) { 718 ret = -ENOMEM; 719 goto out; 720 } 721 do { 722 pte = psb_mmu_mask_pte(start_pfn++, type); 723 psb_mmu_set_pte(pt, addr, pte); 724 pt->count++; 725 } while (addr += PAGE_SIZE, addr < next); 726 psb_mmu_pt_unmap_unlock(pt); 727 728 } while (addr = next, next != end); 729 730 out: 731 if (pd->hw_context != -1) 732 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 733 734 up_read(&pd->driver->sem); 735 736 if (pd->hw_context != -1) 737 psb_mmu_flush(pd->driver, 1); 738 739 return ret; 740 } 741 742 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 743 unsigned long address, uint32_t num_pages, 744 uint32_t desired_tile_stride, 745 uint32_t hw_tile_stride, int type) 746 { 747 struct psb_mmu_pt *pt; 748 uint32_t rows = 1; 749 uint32_t i; 750 uint32_t pte; 751 unsigned long addr; 752 unsigned long end; 753 unsigned long next; 754 unsigned long add; 755 unsigned long row_add; 756 unsigned long f_address = address; 757 int ret = 0; 758 759 if (hw_tile_stride) { 760 if (num_pages % desired_tile_stride != 0) 761 return -EINVAL; 762 rows = num_pages / desired_tile_stride; 763 } else { 764 desired_tile_stride = num_pages; 765 } 766 767 add = desired_tile_stride << PAGE_SHIFT; 768 row_add = hw_tile_stride << PAGE_SHIFT; 769 770 down_read(&pd->driver->sem); 771 772 for (i = 0; i < rows; ++i) { 773 774 addr = address; 775 end = addr + add; 776 777 do { 778 next = psb_pd_addr_end(addr, end); 779 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 780 if (!pt) { 781 ret = -ENOMEM; 782 goto out; 783 } 784 do { 785 pte = 786 psb_mmu_mask_pte(page_to_pfn(*pages++), 787 type); 788 psb_mmu_set_pte(pt, addr, pte); 789 pt->count++; 790 } while (addr += PAGE_SIZE, addr < next); 791 psb_mmu_pt_unmap_unlock(pt); 792 793 } while (addr = next, next != end); 794 795 address += row_add; 796 } 797 out: 798 if (pd->hw_context != -1) 799 psb_mmu_flush_ptes(pd, f_address, num_pages, 800 desired_tile_stride, hw_tile_stride); 801 802 up_read(&pd->driver->sem); 803 804 if (pd->hw_context != -1) 805 psb_mmu_flush(pd->driver, 1); 806 807 return ret; 808 } 809 810 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 811 unsigned long *pfn) 812 { 813 int ret; 814 struct psb_mmu_pt *pt; 815 uint32_t tmp; 816 spinlock_t *lock = &pd->driver->lock; 817 818 down_read(&pd->driver->sem); 819 pt = psb_mmu_pt_map_lock(pd, virtual); 820 if (!pt) { 821 uint32_t *v; 822 823 spin_lock(lock); 824 v = kmap_atomic(pd->p); 825 tmp = v[psb_mmu_pd_index(virtual)]; 826 kunmap_atomic(v); 827 spin_unlock(lock); 828 829 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 830 !(pd->invalid_pte & PSB_PTE_VALID)) { 831 ret = -EINVAL; 832 goto out; 833 } 834 ret = 0; 835 *pfn = pd->invalid_pte >> PAGE_SHIFT; 836 goto out; 837 } 838 tmp = pt->v[psb_mmu_pt_index(virtual)]; 839 if (!(tmp & PSB_PTE_VALID)) { 840 ret = -EINVAL; 841 } else { 842 ret = 0; 843 *pfn = tmp >> PAGE_SHIFT; 844 } 845 psb_mmu_pt_unmap_unlock(pt); 846 out: 847 up_read(&pd->driver->sem); 848 return ret; 849 } 850