1 // SPDX-License-Identifier: GPL-2.0-only 2 /************************************************************************** 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 **************************************************************************/ 6 #include <drm/drmP.h> 7 #include "psb_drv.h" 8 #include "psb_reg.h" 9 #include "mmu.h" 10 11 /* 12 * Code for the SGX MMU: 13 */ 14 15 /* 16 * clflush on one processor only: 17 * clflush should apparently flush the cache line on all processors in an 18 * SMP system. 19 */ 20 21 /* 22 * kmap atomic: 23 * The usage of the slots must be completely encapsulated within a spinlock, and 24 * no other functions that may be using the locks for other purposed may be 25 * called from within the locked region. 26 * Since the slots are per processor, this will guarantee that we are the only 27 * user. 28 */ 29 30 /* 31 * TODO: Inserting ptes from an interrupt handler: 32 * This may be desirable for some SGX functionality where the GPU can fault in 33 * needed pages. For that, we need to make an atomic insert_pages function, that 34 * may fail. 35 * If it fails, the caller need to insert the page using a workqueue function, 36 * but on average it should be fast. 37 */ 38 39 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 40 { 41 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 42 } 43 44 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 45 { 46 return offset >> PSB_PDE_SHIFT; 47 } 48 49 #if defined(CONFIG_X86) 50 static inline void psb_clflush(void *addr) 51 { 52 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 53 } 54 55 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 56 { 57 if (!driver->has_clflush) 58 return; 59 60 mb(); 61 psb_clflush(addr); 62 mb(); 63 } 64 #else 65 66 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 67 {; 68 } 69 70 #endif 71 72 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force) 73 { 74 struct drm_device *dev = driver->dev; 75 struct drm_psb_private *dev_priv = dev->dev_private; 76 77 if (atomic_read(&driver->needs_tlbflush) || force) { 78 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL); 79 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 80 81 /* Make sure data cache is turned off before enabling it */ 82 wmb(); 83 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 84 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 85 if (driver->msvdx_mmu_invaldc) 86 atomic_set(driver->msvdx_mmu_invaldc, 1); 87 } 88 atomic_set(&driver->needs_tlbflush, 0); 89 } 90 91 #if 0 92 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 93 { 94 down_write(&driver->sem); 95 psb_mmu_flush_pd_locked(driver, force); 96 up_write(&driver->sem); 97 } 98 #endif 99 100 void psb_mmu_flush(struct psb_mmu_driver *driver) 101 { 102 struct drm_device *dev = driver->dev; 103 struct drm_psb_private *dev_priv = dev->dev_private; 104 uint32_t val; 105 106 down_write(&driver->sem); 107 val = PSB_RSGX32(PSB_CR_BIF_CTRL); 108 if (atomic_read(&driver->needs_tlbflush)) 109 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 110 else 111 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL); 112 113 /* Make sure data cache is turned off and MMU is flushed before 114 restoring bank interface control register */ 115 wmb(); 116 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC), 117 PSB_CR_BIF_CTRL); 118 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 119 120 atomic_set(&driver->needs_tlbflush, 0); 121 if (driver->msvdx_mmu_invaldc) 122 atomic_set(driver->msvdx_mmu_invaldc, 1); 123 up_write(&driver->sem); 124 } 125 126 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 127 { 128 struct drm_device *dev = pd->driver->dev; 129 struct drm_psb_private *dev_priv = dev->dev_private; 130 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 : 131 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4; 132 133 down_write(&pd->driver->sem); 134 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset); 135 wmb(); 136 psb_mmu_flush_pd_locked(pd->driver, 1); 137 pd->hw_context = hw_context; 138 up_write(&pd->driver->sem); 139 140 } 141 142 static inline unsigned long psb_pd_addr_end(unsigned long addr, 143 unsigned long end) 144 { 145 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 146 return (addr < end) ? addr : end; 147 } 148 149 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 150 { 151 uint32_t mask = PSB_PTE_VALID; 152 153 if (type & PSB_MMU_CACHED_MEMORY) 154 mask |= PSB_PTE_CACHED; 155 if (type & PSB_MMU_RO_MEMORY) 156 mask |= PSB_PTE_RO; 157 if (type & PSB_MMU_WO_MEMORY) 158 mask |= PSB_PTE_WO; 159 160 return (pfn << PAGE_SHIFT) | mask; 161 } 162 163 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 164 int trap_pagefaults, int invalid_type) 165 { 166 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 167 uint32_t *v; 168 int i; 169 170 if (!pd) 171 return NULL; 172 173 pd->p = alloc_page(GFP_DMA32); 174 if (!pd->p) 175 goto out_err1; 176 pd->dummy_pt = alloc_page(GFP_DMA32); 177 if (!pd->dummy_pt) 178 goto out_err2; 179 pd->dummy_page = alloc_page(GFP_DMA32); 180 if (!pd->dummy_page) 181 goto out_err3; 182 183 if (!trap_pagefaults) { 184 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 185 invalid_type); 186 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 187 invalid_type); 188 } else { 189 pd->invalid_pde = 0; 190 pd->invalid_pte = 0; 191 } 192 193 v = kmap(pd->dummy_pt); 194 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 195 v[i] = pd->invalid_pte; 196 197 kunmap(pd->dummy_pt); 198 199 v = kmap(pd->p); 200 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 201 v[i] = pd->invalid_pde; 202 203 kunmap(pd->p); 204 205 clear_page(kmap(pd->dummy_page)); 206 kunmap(pd->dummy_page); 207 208 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 209 if (!pd->tables) 210 goto out_err4; 211 212 pd->hw_context = -1; 213 pd->pd_mask = PSB_PTE_VALID; 214 pd->driver = driver; 215 216 return pd; 217 218 out_err4: 219 __free_page(pd->dummy_page); 220 out_err3: 221 __free_page(pd->dummy_pt); 222 out_err2: 223 __free_page(pd->p); 224 out_err1: 225 kfree(pd); 226 return NULL; 227 } 228 229 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 230 { 231 __free_page(pt->p); 232 kfree(pt); 233 } 234 235 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 236 { 237 struct psb_mmu_driver *driver = pd->driver; 238 struct drm_device *dev = driver->dev; 239 struct drm_psb_private *dev_priv = dev->dev_private; 240 struct psb_mmu_pt *pt; 241 int i; 242 243 down_write(&driver->sem); 244 if (pd->hw_context != -1) { 245 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4); 246 psb_mmu_flush_pd_locked(driver, 1); 247 } 248 249 /* Should take the spinlock here, but we don't need to do that 250 since we have the semaphore in write mode. */ 251 252 for (i = 0; i < 1024; ++i) { 253 pt = pd->tables[i]; 254 if (pt) 255 psb_mmu_free_pt(pt); 256 } 257 258 vfree(pd->tables); 259 __free_page(pd->dummy_page); 260 __free_page(pd->dummy_pt); 261 __free_page(pd->p); 262 kfree(pd); 263 up_write(&driver->sem); 264 } 265 266 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 267 { 268 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 269 void *v; 270 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 271 uint32_t clflush_count = PAGE_SIZE / clflush_add; 272 spinlock_t *lock = &pd->driver->lock; 273 uint8_t *clf; 274 uint32_t *ptes; 275 int i; 276 277 if (!pt) 278 return NULL; 279 280 pt->p = alloc_page(GFP_DMA32); 281 if (!pt->p) { 282 kfree(pt); 283 return NULL; 284 } 285 286 spin_lock(lock); 287 288 v = kmap_atomic(pt->p); 289 clf = (uint8_t *) v; 290 ptes = (uint32_t *) v; 291 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 292 *ptes++ = pd->invalid_pte; 293 294 #if defined(CONFIG_X86) 295 if (pd->driver->has_clflush && pd->hw_context != -1) { 296 mb(); 297 for (i = 0; i < clflush_count; ++i) { 298 psb_clflush(clf); 299 clf += clflush_add; 300 } 301 mb(); 302 } 303 #endif 304 kunmap_atomic(v); 305 spin_unlock(lock); 306 307 pt->count = 0; 308 pt->pd = pd; 309 pt->index = 0; 310 311 return pt; 312 } 313 314 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 315 unsigned long addr) 316 { 317 uint32_t index = psb_mmu_pd_index(addr); 318 struct psb_mmu_pt *pt; 319 uint32_t *v; 320 spinlock_t *lock = &pd->driver->lock; 321 322 spin_lock(lock); 323 pt = pd->tables[index]; 324 while (!pt) { 325 spin_unlock(lock); 326 pt = psb_mmu_alloc_pt(pd); 327 if (!pt) 328 return NULL; 329 spin_lock(lock); 330 331 if (pd->tables[index]) { 332 spin_unlock(lock); 333 psb_mmu_free_pt(pt); 334 spin_lock(lock); 335 pt = pd->tables[index]; 336 continue; 337 } 338 339 v = kmap_atomic(pd->p); 340 pd->tables[index] = pt; 341 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 342 pt->index = index; 343 kunmap_atomic((void *) v); 344 345 if (pd->hw_context != -1) { 346 psb_mmu_clflush(pd->driver, (void *)&v[index]); 347 atomic_set(&pd->driver->needs_tlbflush, 1); 348 } 349 } 350 pt->v = kmap_atomic(pt->p); 351 return pt; 352 } 353 354 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 355 unsigned long addr) 356 { 357 uint32_t index = psb_mmu_pd_index(addr); 358 struct psb_mmu_pt *pt; 359 spinlock_t *lock = &pd->driver->lock; 360 361 spin_lock(lock); 362 pt = pd->tables[index]; 363 if (!pt) { 364 spin_unlock(lock); 365 return NULL; 366 } 367 pt->v = kmap_atomic(pt->p); 368 return pt; 369 } 370 371 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 372 { 373 struct psb_mmu_pd *pd = pt->pd; 374 uint32_t *v; 375 376 kunmap_atomic(pt->v); 377 if (pt->count == 0) { 378 v = kmap_atomic(pd->p); 379 v[pt->index] = pd->invalid_pde; 380 pd->tables[pt->index] = NULL; 381 382 if (pd->hw_context != -1) { 383 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]); 384 atomic_set(&pd->driver->needs_tlbflush, 1); 385 } 386 kunmap_atomic(v); 387 spin_unlock(&pd->driver->lock); 388 psb_mmu_free_pt(pt); 389 return; 390 } 391 spin_unlock(&pd->driver->lock); 392 } 393 394 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr, 395 uint32_t pte) 396 { 397 pt->v[psb_mmu_pt_index(addr)] = pte; 398 } 399 400 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 401 unsigned long addr) 402 { 403 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 404 } 405 406 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 407 { 408 struct psb_mmu_pd *pd; 409 410 down_read(&driver->sem); 411 pd = driver->default_pd; 412 up_read(&driver->sem); 413 414 return pd; 415 } 416 417 /* Returns the physical address of the PD shared by sgx/msvdx */ 418 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver) 419 { 420 struct psb_mmu_pd *pd; 421 422 pd = psb_mmu_get_default_pd(driver); 423 return page_to_pfn(pd->p) << PAGE_SHIFT; 424 } 425 426 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 427 { 428 struct drm_device *dev = driver->dev; 429 struct drm_psb_private *dev_priv = dev->dev_private; 430 431 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL); 432 psb_mmu_free_pagedir(driver->default_pd); 433 kfree(driver); 434 } 435 436 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev, 437 int trap_pagefaults, 438 int invalid_type, 439 atomic_t *msvdx_mmu_invaldc) 440 { 441 struct psb_mmu_driver *driver; 442 struct drm_psb_private *dev_priv = dev->dev_private; 443 444 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 445 446 if (!driver) 447 return NULL; 448 449 driver->dev = dev; 450 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 451 invalid_type); 452 if (!driver->default_pd) 453 goto out_err1; 454 455 spin_lock_init(&driver->lock); 456 init_rwsem(&driver->sem); 457 down_write(&driver->sem); 458 atomic_set(&driver->needs_tlbflush, 1); 459 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc; 460 461 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL); 462 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT, 463 PSB_CR_BIF_CTRL); 464 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT, 465 PSB_CR_BIF_CTRL); 466 467 driver->has_clflush = 0; 468 469 #if defined(CONFIG_X86) 470 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) { 471 uint32_t tfms, misc, cap0, cap4, clflush_size; 472 473 /* 474 * clflush size is determined at kernel setup for x86_64 but not 475 * for i386. We have to do it here. 476 */ 477 478 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 479 clflush_size = ((misc >> 8) & 0xff) * 8; 480 driver->has_clflush = 1; 481 driver->clflush_add = 482 PAGE_SIZE * clflush_size / sizeof(uint32_t); 483 driver->clflush_mask = driver->clflush_add - 1; 484 driver->clflush_mask = ~driver->clflush_mask; 485 } 486 #endif 487 488 up_write(&driver->sem); 489 return driver; 490 491 out_err1: 492 kfree(driver); 493 return NULL; 494 } 495 496 #if defined(CONFIG_X86) 497 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 498 uint32_t num_pages, uint32_t desired_tile_stride, 499 uint32_t hw_tile_stride) 500 { 501 struct psb_mmu_pt *pt; 502 uint32_t rows = 1; 503 uint32_t i; 504 unsigned long addr; 505 unsigned long end; 506 unsigned long next; 507 unsigned long add; 508 unsigned long row_add; 509 unsigned long clflush_add = pd->driver->clflush_add; 510 unsigned long clflush_mask = pd->driver->clflush_mask; 511 512 if (!pd->driver->has_clflush) 513 return; 514 515 if (hw_tile_stride) 516 rows = num_pages / desired_tile_stride; 517 else 518 desired_tile_stride = num_pages; 519 520 add = desired_tile_stride << PAGE_SHIFT; 521 row_add = hw_tile_stride << PAGE_SHIFT; 522 mb(); 523 for (i = 0; i < rows; ++i) { 524 525 addr = address; 526 end = addr + add; 527 528 do { 529 next = psb_pd_addr_end(addr, end); 530 pt = psb_mmu_pt_map_lock(pd, addr); 531 if (!pt) 532 continue; 533 do { 534 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]); 535 } while (addr += clflush_add, 536 (addr & clflush_mask) < next); 537 538 psb_mmu_pt_unmap_unlock(pt); 539 } while (addr = next, next != end); 540 address += row_add; 541 } 542 mb(); 543 } 544 #else 545 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 546 uint32_t num_pages, uint32_t desired_tile_stride, 547 uint32_t hw_tile_stride) 548 { 549 drm_ttm_cache_flush(); 550 } 551 #endif 552 553 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 554 unsigned long address, uint32_t num_pages) 555 { 556 struct psb_mmu_pt *pt; 557 unsigned long addr; 558 unsigned long end; 559 unsigned long next; 560 unsigned long f_address = address; 561 562 down_read(&pd->driver->sem); 563 564 addr = address; 565 end = addr + (num_pages << PAGE_SHIFT); 566 567 do { 568 next = psb_pd_addr_end(addr, end); 569 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 570 if (!pt) 571 goto out; 572 do { 573 psb_mmu_invalidate_pte(pt, addr); 574 --pt->count; 575 } while (addr += PAGE_SIZE, addr < next); 576 psb_mmu_pt_unmap_unlock(pt); 577 578 } while (addr = next, next != end); 579 580 out: 581 if (pd->hw_context != -1) 582 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 583 584 up_read(&pd->driver->sem); 585 586 if (pd->hw_context != -1) 587 psb_mmu_flush(pd->driver); 588 589 return; 590 } 591 592 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 593 uint32_t num_pages, uint32_t desired_tile_stride, 594 uint32_t hw_tile_stride) 595 { 596 struct psb_mmu_pt *pt; 597 uint32_t rows = 1; 598 uint32_t i; 599 unsigned long addr; 600 unsigned long end; 601 unsigned long next; 602 unsigned long add; 603 unsigned long row_add; 604 unsigned long f_address = address; 605 606 if (hw_tile_stride) 607 rows = num_pages / desired_tile_stride; 608 else 609 desired_tile_stride = num_pages; 610 611 add = desired_tile_stride << PAGE_SHIFT; 612 row_add = hw_tile_stride << PAGE_SHIFT; 613 614 down_read(&pd->driver->sem); 615 616 /* Make sure we only need to flush this processor's cache */ 617 618 for (i = 0; i < rows; ++i) { 619 620 addr = address; 621 end = addr + add; 622 623 do { 624 next = psb_pd_addr_end(addr, end); 625 pt = psb_mmu_pt_map_lock(pd, addr); 626 if (!pt) 627 continue; 628 do { 629 psb_mmu_invalidate_pte(pt, addr); 630 --pt->count; 631 632 } while (addr += PAGE_SIZE, addr < next); 633 psb_mmu_pt_unmap_unlock(pt); 634 635 } while (addr = next, next != end); 636 address += row_add; 637 } 638 if (pd->hw_context != -1) 639 psb_mmu_flush_ptes(pd, f_address, num_pages, 640 desired_tile_stride, hw_tile_stride); 641 642 up_read(&pd->driver->sem); 643 644 if (pd->hw_context != -1) 645 psb_mmu_flush(pd->driver); 646 } 647 648 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 649 unsigned long address, uint32_t num_pages, 650 int type) 651 { 652 struct psb_mmu_pt *pt; 653 uint32_t pte; 654 unsigned long addr; 655 unsigned long end; 656 unsigned long next; 657 unsigned long f_address = address; 658 int ret = -ENOMEM; 659 660 down_read(&pd->driver->sem); 661 662 addr = address; 663 end = addr + (num_pages << PAGE_SHIFT); 664 665 do { 666 next = psb_pd_addr_end(addr, end); 667 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 668 if (!pt) { 669 ret = -ENOMEM; 670 goto out; 671 } 672 do { 673 pte = psb_mmu_mask_pte(start_pfn++, type); 674 psb_mmu_set_pte(pt, addr, pte); 675 pt->count++; 676 } while (addr += PAGE_SIZE, addr < next); 677 psb_mmu_pt_unmap_unlock(pt); 678 679 } while (addr = next, next != end); 680 ret = 0; 681 682 out: 683 if (pd->hw_context != -1) 684 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 685 686 up_read(&pd->driver->sem); 687 688 if (pd->hw_context != -1) 689 psb_mmu_flush(pd->driver); 690 691 return 0; 692 } 693 694 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 695 unsigned long address, uint32_t num_pages, 696 uint32_t desired_tile_stride, uint32_t hw_tile_stride, 697 int type) 698 { 699 struct psb_mmu_pt *pt; 700 uint32_t rows = 1; 701 uint32_t i; 702 uint32_t pte; 703 unsigned long addr; 704 unsigned long end; 705 unsigned long next; 706 unsigned long add; 707 unsigned long row_add; 708 unsigned long f_address = address; 709 int ret = -ENOMEM; 710 711 if (hw_tile_stride) { 712 if (num_pages % desired_tile_stride != 0) 713 return -EINVAL; 714 rows = num_pages / desired_tile_stride; 715 } else { 716 desired_tile_stride = num_pages; 717 } 718 719 add = desired_tile_stride << PAGE_SHIFT; 720 row_add = hw_tile_stride << PAGE_SHIFT; 721 722 down_read(&pd->driver->sem); 723 724 for (i = 0; i < rows; ++i) { 725 726 addr = address; 727 end = addr + add; 728 729 do { 730 next = psb_pd_addr_end(addr, end); 731 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 732 if (!pt) 733 goto out; 734 do { 735 pte = psb_mmu_mask_pte(page_to_pfn(*pages++), 736 type); 737 psb_mmu_set_pte(pt, addr, pte); 738 pt->count++; 739 } while (addr += PAGE_SIZE, addr < next); 740 psb_mmu_pt_unmap_unlock(pt); 741 742 } while (addr = next, next != end); 743 744 address += row_add; 745 } 746 747 ret = 0; 748 out: 749 if (pd->hw_context != -1) 750 psb_mmu_flush_ptes(pd, f_address, num_pages, 751 desired_tile_stride, hw_tile_stride); 752 753 up_read(&pd->driver->sem); 754 755 if (pd->hw_context != -1) 756 psb_mmu_flush(pd->driver); 757 758 return ret; 759 } 760 761 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 762 unsigned long *pfn) 763 { 764 int ret; 765 struct psb_mmu_pt *pt; 766 uint32_t tmp; 767 spinlock_t *lock = &pd->driver->lock; 768 769 down_read(&pd->driver->sem); 770 pt = psb_mmu_pt_map_lock(pd, virtual); 771 if (!pt) { 772 uint32_t *v; 773 774 spin_lock(lock); 775 v = kmap_atomic(pd->p); 776 tmp = v[psb_mmu_pd_index(virtual)]; 777 kunmap_atomic(v); 778 spin_unlock(lock); 779 780 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 781 !(pd->invalid_pte & PSB_PTE_VALID)) { 782 ret = -EINVAL; 783 goto out; 784 } 785 ret = 0; 786 *pfn = pd->invalid_pte >> PAGE_SHIFT; 787 goto out; 788 } 789 tmp = pt->v[psb_mmu_pt_index(virtual)]; 790 if (!(tmp & PSB_PTE_VALID)) { 791 ret = -EINVAL; 792 } else { 793 ret = 0; 794 *pfn = tmp >> PAGE_SHIFT; 795 } 796 psb_mmu_pt_unmap_unlock(pt); 797 out: 798 up_read(&pd->driver->sem); 799 return ret; 800 } 801