1 // SPDX-License-Identifier: GPL-2.0-only 2 /************************************************************************** 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 **************************************************************************/ 6 7 #include <linux/highmem.h> 8 9 #include "mmu.h" 10 #include "psb_drv.h" 11 #include "psb_reg.h" 12 13 /* 14 * Code for the SGX MMU: 15 */ 16 17 /* 18 * clflush on one processor only: 19 * clflush should apparently flush the cache line on all processors in an 20 * SMP system. 21 */ 22 23 /* 24 * kmap atomic: 25 * The usage of the slots must be completely encapsulated within a spinlock, and 26 * no other functions that may be using the locks for other purposed may be 27 * called from within the locked region. 28 * Since the slots are per processor, this will guarantee that we are the only 29 * user. 30 */ 31 32 /* 33 * TODO: Inserting ptes from an interrupt handler: 34 * This may be desirable for some SGX functionality where the GPU can fault in 35 * needed pages. For that, we need to make an atomic insert_pages function, that 36 * may fail. 37 * If it fails, the caller need to insert the page using a workqueue function, 38 * but on average it should be fast. 39 */ 40 41 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 42 { 43 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 44 } 45 46 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 47 { 48 return offset >> PSB_PDE_SHIFT; 49 } 50 51 #if defined(CONFIG_X86) 52 static inline void psb_clflush(void *addr) 53 { 54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 55 } 56 57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 58 { 59 if (!driver->has_clflush) 60 return; 61 62 mb(); 63 psb_clflush(addr); 64 mb(); 65 } 66 #else 67 68 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 69 {; 70 } 71 72 #endif 73 74 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force) 75 { 76 struct drm_device *dev = driver->dev; 77 struct drm_psb_private *dev_priv = dev->dev_private; 78 79 if (atomic_read(&driver->needs_tlbflush) || force) { 80 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL); 81 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 82 83 /* Make sure data cache is turned off before enabling it */ 84 wmb(); 85 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 86 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 87 if (driver->msvdx_mmu_invaldc) 88 atomic_set(driver->msvdx_mmu_invaldc, 1); 89 } 90 atomic_set(&driver->needs_tlbflush, 0); 91 } 92 93 #if 0 94 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 95 { 96 down_write(&driver->sem); 97 psb_mmu_flush_pd_locked(driver, force); 98 up_write(&driver->sem); 99 } 100 #endif 101 102 void psb_mmu_flush(struct psb_mmu_driver *driver) 103 { 104 struct drm_device *dev = driver->dev; 105 struct drm_psb_private *dev_priv = dev->dev_private; 106 uint32_t val; 107 108 down_write(&driver->sem); 109 val = PSB_RSGX32(PSB_CR_BIF_CTRL); 110 if (atomic_read(&driver->needs_tlbflush)) 111 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 112 else 113 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL); 114 115 /* Make sure data cache is turned off and MMU is flushed before 116 restoring bank interface control register */ 117 wmb(); 118 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC), 119 PSB_CR_BIF_CTRL); 120 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 121 122 atomic_set(&driver->needs_tlbflush, 0); 123 if (driver->msvdx_mmu_invaldc) 124 atomic_set(driver->msvdx_mmu_invaldc, 1); 125 up_write(&driver->sem); 126 } 127 128 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 129 { 130 struct drm_device *dev = pd->driver->dev; 131 struct drm_psb_private *dev_priv = dev->dev_private; 132 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 : 133 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4; 134 135 down_write(&pd->driver->sem); 136 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset); 137 wmb(); 138 psb_mmu_flush_pd_locked(pd->driver, 1); 139 pd->hw_context = hw_context; 140 up_write(&pd->driver->sem); 141 142 } 143 144 static inline unsigned long psb_pd_addr_end(unsigned long addr, 145 unsigned long end) 146 { 147 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 148 return (addr < end) ? addr : end; 149 } 150 151 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 152 { 153 uint32_t mask = PSB_PTE_VALID; 154 155 if (type & PSB_MMU_CACHED_MEMORY) 156 mask |= PSB_PTE_CACHED; 157 if (type & PSB_MMU_RO_MEMORY) 158 mask |= PSB_PTE_RO; 159 if (type & PSB_MMU_WO_MEMORY) 160 mask |= PSB_PTE_WO; 161 162 return (pfn << PAGE_SHIFT) | mask; 163 } 164 165 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 166 int trap_pagefaults, int invalid_type) 167 { 168 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 169 uint32_t *v; 170 int i; 171 172 if (!pd) 173 return NULL; 174 175 pd->p = alloc_page(GFP_DMA32); 176 if (!pd->p) 177 goto out_err1; 178 pd->dummy_pt = alloc_page(GFP_DMA32); 179 if (!pd->dummy_pt) 180 goto out_err2; 181 pd->dummy_page = alloc_page(GFP_DMA32); 182 if (!pd->dummy_page) 183 goto out_err3; 184 185 if (!trap_pagefaults) { 186 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 187 invalid_type); 188 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 189 invalid_type); 190 } else { 191 pd->invalid_pde = 0; 192 pd->invalid_pte = 0; 193 } 194 195 v = kmap(pd->dummy_pt); 196 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 197 v[i] = pd->invalid_pte; 198 199 kunmap(pd->dummy_pt); 200 201 v = kmap(pd->p); 202 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 203 v[i] = pd->invalid_pde; 204 205 kunmap(pd->p); 206 207 clear_page(kmap(pd->dummy_page)); 208 kunmap(pd->dummy_page); 209 210 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 211 if (!pd->tables) 212 goto out_err4; 213 214 pd->hw_context = -1; 215 pd->pd_mask = PSB_PTE_VALID; 216 pd->driver = driver; 217 218 return pd; 219 220 out_err4: 221 __free_page(pd->dummy_page); 222 out_err3: 223 __free_page(pd->dummy_pt); 224 out_err2: 225 __free_page(pd->p); 226 out_err1: 227 kfree(pd); 228 return NULL; 229 } 230 231 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 232 { 233 __free_page(pt->p); 234 kfree(pt); 235 } 236 237 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 238 { 239 struct psb_mmu_driver *driver = pd->driver; 240 struct drm_device *dev = driver->dev; 241 struct drm_psb_private *dev_priv = dev->dev_private; 242 struct psb_mmu_pt *pt; 243 int i; 244 245 down_write(&driver->sem); 246 if (pd->hw_context != -1) { 247 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4); 248 psb_mmu_flush_pd_locked(driver, 1); 249 } 250 251 /* Should take the spinlock here, but we don't need to do that 252 since we have the semaphore in write mode. */ 253 254 for (i = 0; i < 1024; ++i) { 255 pt = pd->tables[i]; 256 if (pt) 257 psb_mmu_free_pt(pt); 258 } 259 260 vfree(pd->tables); 261 __free_page(pd->dummy_page); 262 __free_page(pd->dummy_pt); 263 __free_page(pd->p); 264 kfree(pd); 265 up_write(&driver->sem); 266 } 267 268 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 269 { 270 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 271 void *v; 272 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 273 uint32_t clflush_count = PAGE_SIZE / clflush_add; 274 spinlock_t *lock = &pd->driver->lock; 275 uint8_t *clf; 276 uint32_t *ptes; 277 int i; 278 279 if (!pt) 280 return NULL; 281 282 pt->p = alloc_page(GFP_DMA32); 283 if (!pt->p) { 284 kfree(pt); 285 return NULL; 286 } 287 288 spin_lock(lock); 289 290 v = kmap_atomic(pt->p); 291 clf = (uint8_t *) v; 292 ptes = (uint32_t *) v; 293 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 294 *ptes++ = pd->invalid_pte; 295 296 #if defined(CONFIG_X86) 297 if (pd->driver->has_clflush && pd->hw_context != -1) { 298 mb(); 299 for (i = 0; i < clflush_count; ++i) { 300 psb_clflush(clf); 301 clf += clflush_add; 302 } 303 mb(); 304 } 305 #endif 306 kunmap_atomic(v); 307 spin_unlock(lock); 308 309 pt->count = 0; 310 pt->pd = pd; 311 pt->index = 0; 312 313 return pt; 314 } 315 316 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 317 unsigned long addr) 318 { 319 uint32_t index = psb_mmu_pd_index(addr); 320 struct psb_mmu_pt *pt; 321 uint32_t *v; 322 spinlock_t *lock = &pd->driver->lock; 323 324 spin_lock(lock); 325 pt = pd->tables[index]; 326 while (!pt) { 327 spin_unlock(lock); 328 pt = psb_mmu_alloc_pt(pd); 329 if (!pt) 330 return NULL; 331 spin_lock(lock); 332 333 if (pd->tables[index]) { 334 spin_unlock(lock); 335 psb_mmu_free_pt(pt); 336 spin_lock(lock); 337 pt = pd->tables[index]; 338 continue; 339 } 340 341 v = kmap_atomic(pd->p); 342 pd->tables[index] = pt; 343 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 344 pt->index = index; 345 kunmap_atomic((void *) v); 346 347 if (pd->hw_context != -1) { 348 psb_mmu_clflush(pd->driver, (void *)&v[index]); 349 atomic_set(&pd->driver->needs_tlbflush, 1); 350 } 351 } 352 pt->v = kmap_atomic(pt->p); 353 return pt; 354 } 355 356 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 357 unsigned long addr) 358 { 359 uint32_t index = psb_mmu_pd_index(addr); 360 struct psb_mmu_pt *pt; 361 spinlock_t *lock = &pd->driver->lock; 362 363 spin_lock(lock); 364 pt = pd->tables[index]; 365 if (!pt) { 366 spin_unlock(lock); 367 return NULL; 368 } 369 pt->v = kmap_atomic(pt->p); 370 return pt; 371 } 372 373 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 374 { 375 struct psb_mmu_pd *pd = pt->pd; 376 uint32_t *v; 377 378 kunmap_atomic(pt->v); 379 if (pt->count == 0) { 380 v = kmap_atomic(pd->p); 381 v[pt->index] = pd->invalid_pde; 382 pd->tables[pt->index] = NULL; 383 384 if (pd->hw_context != -1) { 385 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]); 386 atomic_set(&pd->driver->needs_tlbflush, 1); 387 } 388 kunmap_atomic(v); 389 spin_unlock(&pd->driver->lock); 390 psb_mmu_free_pt(pt); 391 return; 392 } 393 spin_unlock(&pd->driver->lock); 394 } 395 396 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr, 397 uint32_t pte) 398 { 399 pt->v[psb_mmu_pt_index(addr)] = pte; 400 } 401 402 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 403 unsigned long addr) 404 { 405 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 406 } 407 408 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 409 { 410 struct psb_mmu_pd *pd; 411 412 down_read(&driver->sem); 413 pd = driver->default_pd; 414 up_read(&driver->sem); 415 416 return pd; 417 } 418 419 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 420 { 421 struct drm_device *dev = driver->dev; 422 struct drm_psb_private *dev_priv = dev->dev_private; 423 424 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL); 425 psb_mmu_free_pagedir(driver->default_pd); 426 kfree(driver); 427 } 428 429 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev, 430 int trap_pagefaults, 431 int invalid_type, 432 atomic_t *msvdx_mmu_invaldc) 433 { 434 struct psb_mmu_driver *driver; 435 struct drm_psb_private *dev_priv = dev->dev_private; 436 437 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 438 439 if (!driver) 440 return NULL; 441 442 driver->dev = dev; 443 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 444 invalid_type); 445 if (!driver->default_pd) 446 goto out_err1; 447 448 spin_lock_init(&driver->lock); 449 init_rwsem(&driver->sem); 450 down_write(&driver->sem); 451 atomic_set(&driver->needs_tlbflush, 1); 452 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc; 453 454 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL); 455 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT, 456 PSB_CR_BIF_CTRL); 457 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT, 458 PSB_CR_BIF_CTRL); 459 460 driver->has_clflush = 0; 461 462 #if defined(CONFIG_X86) 463 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) { 464 uint32_t tfms, misc, cap0, cap4, clflush_size; 465 466 /* 467 * clflush size is determined at kernel setup for x86_64 but not 468 * for i386. We have to do it here. 469 */ 470 471 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 472 clflush_size = ((misc >> 8) & 0xff) * 8; 473 driver->has_clflush = 1; 474 driver->clflush_add = 475 PAGE_SIZE * clflush_size / sizeof(uint32_t); 476 driver->clflush_mask = driver->clflush_add - 1; 477 driver->clflush_mask = ~driver->clflush_mask; 478 } 479 #endif 480 481 up_write(&driver->sem); 482 return driver; 483 484 out_err1: 485 kfree(driver); 486 return NULL; 487 } 488 489 #if defined(CONFIG_X86) 490 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 491 uint32_t num_pages, uint32_t desired_tile_stride, 492 uint32_t hw_tile_stride) 493 { 494 struct psb_mmu_pt *pt; 495 uint32_t rows = 1; 496 uint32_t i; 497 unsigned long addr; 498 unsigned long end; 499 unsigned long next; 500 unsigned long add; 501 unsigned long row_add; 502 unsigned long clflush_add = pd->driver->clflush_add; 503 unsigned long clflush_mask = pd->driver->clflush_mask; 504 505 if (!pd->driver->has_clflush) 506 return; 507 508 if (hw_tile_stride) 509 rows = num_pages / desired_tile_stride; 510 else 511 desired_tile_stride = num_pages; 512 513 add = desired_tile_stride << PAGE_SHIFT; 514 row_add = hw_tile_stride << PAGE_SHIFT; 515 mb(); 516 for (i = 0; i < rows; ++i) { 517 518 addr = address; 519 end = addr + add; 520 521 do { 522 next = psb_pd_addr_end(addr, end); 523 pt = psb_mmu_pt_map_lock(pd, addr); 524 if (!pt) 525 continue; 526 do { 527 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]); 528 } while (addr += clflush_add, 529 (addr & clflush_mask) < next); 530 531 psb_mmu_pt_unmap_unlock(pt); 532 } while (addr = next, next != end); 533 address += row_add; 534 } 535 mb(); 536 } 537 #else 538 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 539 uint32_t num_pages, uint32_t desired_tile_stride, 540 uint32_t hw_tile_stride) 541 { 542 drm_ttm_cache_flush(); 543 } 544 #endif 545 546 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 547 unsigned long address, uint32_t num_pages) 548 { 549 struct psb_mmu_pt *pt; 550 unsigned long addr; 551 unsigned long end; 552 unsigned long next; 553 unsigned long f_address = address; 554 555 down_read(&pd->driver->sem); 556 557 addr = address; 558 end = addr + (num_pages << PAGE_SHIFT); 559 560 do { 561 next = psb_pd_addr_end(addr, end); 562 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 563 if (!pt) 564 goto out; 565 do { 566 psb_mmu_invalidate_pte(pt, addr); 567 --pt->count; 568 } while (addr += PAGE_SIZE, addr < next); 569 psb_mmu_pt_unmap_unlock(pt); 570 571 } while (addr = next, next != end); 572 573 out: 574 if (pd->hw_context != -1) 575 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 576 577 up_read(&pd->driver->sem); 578 579 if (pd->hw_context != -1) 580 psb_mmu_flush(pd->driver); 581 582 return; 583 } 584 585 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 586 uint32_t num_pages, uint32_t desired_tile_stride, 587 uint32_t hw_tile_stride) 588 { 589 struct psb_mmu_pt *pt; 590 uint32_t rows = 1; 591 uint32_t i; 592 unsigned long addr; 593 unsigned long end; 594 unsigned long next; 595 unsigned long add; 596 unsigned long row_add; 597 unsigned long f_address = address; 598 599 if (hw_tile_stride) 600 rows = num_pages / desired_tile_stride; 601 else 602 desired_tile_stride = num_pages; 603 604 add = desired_tile_stride << PAGE_SHIFT; 605 row_add = hw_tile_stride << PAGE_SHIFT; 606 607 down_read(&pd->driver->sem); 608 609 /* Make sure we only need to flush this processor's cache */ 610 611 for (i = 0; i < rows; ++i) { 612 613 addr = address; 614 end = addr + add; 615 616 do { 617 next = psb_pd_addr_end(addr, end); 618 pt = psb_mmu_pt_map_lock(pd, addr); 619 if (!pt) 620 continue; 621 do { 622 psb_mmu_invalidate_pte(pt, addr); 623 --pt->count; 624 625 } while (addr += PAGE_SIZE, addr < next); 626 psb_mmu_pt_unmap_unlock(pt); 627 628 } while (addr = next, next != end); 629 address += row_add; 630 } 631 if (pd->hw_context != -1) 632 psb_mmu_flush_ptes(pd, f_address, num_pages, 633 desired_tile_stride, hw_tile_stride); 634 635 up_read(&pd->driver->sem); 636 637 if (pd->hw_context != -1) 638 psb_mmu_flush(pd->driver); 639 } 640 641 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 642 unsigned long address, uint32_t num_pages, 643 int type) 644 { 645 struct psb_mmu_pt *pt; 646 uint32_t pte; 647 unsigned long addr; 648 unsigned long end; 649 unsigned long next; 650 unsigned long f_address = address; 651 int ret = -ENOMEM; 652 653 down_read(&pd->driver->sem); 654 655 addr = address; 656 end = addr + (num_pages << PAGE_SHIFT); 657 658 do { 659 next = psb_pd_addr_end(addr, end); 660 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 661 if (!pt) { 662 ret = -ENOMEM; 663 goto out; 664 } 665 do { 666 pte = psb_mmu_mask_pte(start_pfn++, type); 667 psb_mmu_set_pte(pt, addr, pte); 668 pt->count++; 669 } while (addr += PAGE_SIZE, addr < next); 670 psb_mmu_pt_unmap_unlock(pt); 671 672 } while (addr = next, next != end); 673 ret = 0; 674 675 out: 676 if (pd->hw_context != -1) 677 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 678 679 up_read(&pd->driver->sem); 680 681 if (pd->hw_context != -1) 682 psb_mmu_flush(pd->driver); 683 684 return ret; 685 } 686 687 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 688 unsigned long address, uint32_t num_pages, 689 uint32_t desired_tile_stride, uint32_t hw_tile_stride, 690 int type) 691 { 692 struct psb_mmu_pt *pt; 693 uint32_t rows = 1; 694 uint32_t i; 695 uint32_t pte; 696 unsigned long addr; 697 unsigned long end; 698 unsigned long next; 699 unsigned long add; 700 unsigned long row_add; 701 unsigned long f_address = address; 702 int ret = -ENOMEM; 703 704 if (hw_tile_stride) { 705 if (num_pages % desired_tile_stride != 0) 706 return -EINVAL; 707 rows = num_pages / desired_tile_stride; 708 } else { 709 desired_tile_stride = num_pages; 710 } 711 712 add = desired_tile_stride << PAGE_SHIFT; 713 row_add = hw_tile_stride << PAGE_SHIFT; 714 715 down_read(&pd->driver->sem); 716 717 for (i = 0; i < rows; ++i) { 718 719 addr = address; 720 end = addr + add; 721 722 do { 723 next = psb_pd_addr_end(addr, end); 724 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 725 if (!pt) 726 goto out; 727 do { 728 pte = psb_mmu_mask_pte(page_to_pfn(*pages++), 729 type); 730 psb_mmu_set_pte(pt, addr, pte); 731 pt->count++; 732 } while (addr += PAGE_SIZE, addr < next); 733 psb_mmu_pt_unmap_unlock(pt); 734 735 } while (addr = next, next != end); 736 737 address += row_add; 738 } 739 740 ret = 0; 741 out: 742 if (pd->hw_context != -1) 743 psb_mmu_flush_ptes(pd, f_address, num_pages, 744 desired_tile_stride, hw_tile_stride); 745 746 up_read(&pd->driver->sem); 747 748 if (pd->hw_context != -1) 749 psb_mmu_flush(pd->driver); 750 751 return ret; 752 } 753 754 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 755 unsigned long *pfn) 756 { 757 int ret; 758 struct psb_mmu_pt *pt; 759 uint32_t tmp; 760 spinlock_t *lock = &pd->driver->lock; 761 762 down_read(&pd->driver->sem); 763 pt = psb_mmu_pt_map_lock(pd, virtual); 764 if (!pt) { 765 uint32_t *v; 766 767 spin_lock(lock); 768 v = kmap_atomic(pd->p); 769 tmp = v[psb_mmu_pd_index(virtual)]; 770 kunmap_atomic(v); 771 spin_unlock(lock); 772 773 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 774 !(pd->invalid_pte & PSB_PTE_VALID)) { 775 ret = -EINVAL; 776 goto out; 777 } 778 ret = 0; 779 *pfn = pd->invalid_pte >> PAGE_SHIFT; 780 goto out; 781 } 782 tmp = pt->v[psb_mmu_pt_index(virtual)]; 783 if (!(tmp & PSB_PTE_VALID)) { 784 ret = -EINVAL; 785 } else { 786 ret = 0; 787 *pfn = tmp >> PAGE_SHIFT; 788 } 789 psb_mmu_pt_unmap_unlock(pt); 790 out: 791 up_read(&pd->driver->sem); 792 return ret; 793 } 794