1 /************************************************************************** 2 * Copyright (c) 2007, Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, write to the Free Software Foundation, Inc., 15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 16 * 17 **************************************************************************/ 18 #include <drm/drmP.h> 19 #include "psb_drv.h" 20 #include "psb_reg.h" 21 #include "mmu.h" 22 23 /* 24 * Code for the SGX MMU: 25 */ 26 27 /* 28 * clflush on one processor only: 29 * clflush should apparently flush the cache line on all processors in an 30 * SMP system. 31 */ 32 33 /* 34 * kmap atomic: 35 * The usage of the slots must be completely encapsulated within a spinlock, and 36 * no other functions that may be using the locks for other purposed may be 37 * called from within the locked region. 38 * Since the slots are per processor, this will guarantee that we are the only 39 * user. 40 */ 41 42 /* 43 * TODO: Inserting ptes from an interrupt handler: 44 * This may be desirable for some SGX functionality where the GPU can fault in 45 * needed pages. For that, we need to make an atomic insert_pages function, that 46 * may fail. 47 * If it fails, the caller need to insert the page using a workqueue function, 48 * but on average it should be fast. 49 */ 50 51 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 52 { 53 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 54 } 55 56 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 57 { 58 return offset >> PSB_PDE_SHIFT; 59 } 60 61 #if defined(CONFIG_X86) 62 static inline void psb_clflush(void *addr) 63 { 64 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 65 } 66 67 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 68 { 69 if (!driver->has_clflush) 70 return; 71 72 mb(); 73 psb_clflush(addr); 74 mb(); 75 } 76 #else 77 78 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 79 {; 80 } 81 82 #endif 83 84 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force) 85 { 86 struct drm_device *dev = driver->dev; 87 struct drm_psb_private *dev_priv = dev->dev_private; 88 89 if (atomic_read(&driver->needs_tlbflush) || force) { 90 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL); 91 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 92 93 /* Make sure data cache is turned off before enabling it */ 94 wmb(); 95 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 96 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 97 if (driver->msvdx_mmu_invaldc) 98 atomic_set(driver->msvdx_mmu_invaldc, 1); 99 } 100 atomic_set(&driver->needs_tlbflush, 0); 101 } 102 103 #if 0 104 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 105 { 106 down_write(&driver->sem); 107 psb_mmu_flush_pd_locked(driver, force); 108 up_write(&driver->sem); 109 } 110 #endif 111 112 void psb_mmu_flush(struct psb_mmu_driver *driver) 113 { 114 struct drm_device *dev = driver->dev; 115 struct drm_psb_private *dev_priv = dev->dev_private; 116 uint32_t val; 117 118 down_write(&driver->sem); 119 val = PSB_RSGX32(PSB_CR_BIF_CTRL); 120 if (atomic_read(&driver->needs_tlbflush)) 121 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 122 else 123 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL); 124 125 /* Make sure data cache is turned off and MMU is flushed before 126 restoring bank interface control register */ 127 wmb(); 128 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC), 129 PSB_CR_BIF_CTRL); 130 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 131 132 atomic_set(&driver->needs_tlbflush, 0); 133 if (driver->msvdx_mmu_invaldc) 134 atomic_set(driver->msvdx_mmu_invaldc, 1); 135 up_write(&driver->sem); 136 } 137 138 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 139 { 140 struct drm_device *dev = pd->driver->dev; 141 struct drm_psb_private *dev_priv = dev->dev_private; 142 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 : 143 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4; 144 145 down_write(&pd->driver->sem); 146 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset); 147 wmb(); 148 psb_mmu_flush_pd_locked(pd->driver, 1); 149 pd->hw_context = hw_context; 150 up_write(&pd->driver->sem); 151 152 } 153 154 static inline unsigned long psb_pd_addr_end(unsigned long addr, 155 unsigned long end) 156 { 157 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 158 return (addr < end) ? addr : end; 159 } 160 161 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 162 { 163 uint32_t mask = PSB_PTE_VALID; 164 165 if (type & PSB_MMU_CACHED_MEMORY) 166 mask |= PSB_PTE_CACHED; 167 if (type & PSB_MMU_RO_MEMORY) 168 mask |= PSB_PTE_RO; 169 if (type & PSB_MMU_WO_MEMORY) 170 mask |= PSB_PTE_WO; 171 172 return (pfn << PAGE_SHIFT) | mask; 173 } 174 175 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 176 int trap_pagefaults, int invalid_type) 177 { 178 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 179 uint32_t *v; 180 int i; 181 182 if (!pd) 183 return NULL; 184 185 pd->p = alloc_page(GFP_DMA32); 186 if (!pd->p) 187 goto out_err1; 188 pd->dummy_pt = alloc_page(GFP_DMA32); 189 if (!pd->dummy_pt) 190 goto out_err2; 191 pd->dummy_page = alloc_page(GFP_DMA32); 192 if (!pd->dummy_page) 193 goto out_err3; 194 195 if (!trap_pagefaults) { 196 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 197 invalid_type); 198 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 199 invalid_type); 200 } else { 201 pd->invalid_pde = 0; 202 pd->invalid_pte = 0; 203 } 204 205 v = kmap(pd->dummy_pt); 206 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 207 v[i] = pd->invalid_pte; 208 209 kunmap(pd->dummy_pt); 210 211 v = kmap(pd->p); 212 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 213 v[i] = pd->invalid_pde; 214 215 kunmap(pd->p); 216 217 clear_page(kmap(pd->dummy_page)); 218 kunmap(pd->dummy_page); 219 220 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 221 if (!pd->tables) 222 goto out_err4; 223 224 pd->hw_context = -1; 225 pd->pd_mask = PSB_PTE_VALID; 226 pd->driver = driver; 227 228 return pd; 229 230 out_err4: 231 __free_page(pd->dummy_page); 232 out_err3: 233 __free_page(pd->dummy_pt); 234 out_err2: 235 __free_page(pd->p); 236 out_err1: 237 kfree(pd); 238 return NULL; 239 } 240 241 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 242 { 243 __free_page(pt->p); 244 kfree(pt); 245 } 246 247 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 248 { 249 struct psb_mmu_driver *driver = pd->driver; 250 struct drm_device *dev = driver->dev; 251 struct drm_psb_private *dev_priv = dev->dev_private; 252 struct psb_mmu_pt *pt; 253 int i; 254 255 down_write(&driver->sem); 256 if (pd->hw_context != -1) { 257 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4); 258 psb_mmu_flush_pd_locked(driver, 1); 259 } 260 261 /* Should take the spinlock here, but we don't need to do that 262 since we have the semaphore in write mode. */ 263 264 for (i = 0; i < 1024; ++i) { 265 pt = pd->tables[i]; 266 if (pt) 267 psb_mmu_free_pt(pt); 268 } 269 270 vfree(pd->tables); 271 __free_page(pd->dummy_page); 272 __free_page(pd->dummy_pt); 273 __free_page(pd->p); 274 kfree(pd); 275 up_write(&driver->sem); 276 } 277 278 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 279 { 280 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 281 void *v; 282 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 283 uint32_t clflush_count = PAGE_SIZE / clflush_add; 284 spinlock_t *lock = &pd->driver->lock; 285 uint8_t *clf; 286 uint32_t *ptes; 287 int i; 288 289 if (!pt) 290 return NULL; 291 292 pt->p = alloc_page(GFP_DMA32); 293 if (!pt->p) { 294 kfree(pt); 295 return NULL; 296 } 297 298 spin_lock(lock); 299 300 v = kmap_atomic(pt->p); 301 clf = (uint8_t *) v; 302 ptes = (uint32_t *) v; 303 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 304 *ptes++ = pd->invalid_pte; 305 306 #if defined(CONFIG_X86) 307 if (pd->driver->has_clflush && pd->hw_context != -1) { 308 mb(); 309 for (i = 0; i < clflush_count; ++i) { 310 psb_clflush(clf); 311 clf += clflush_add; 312 } 313 mb(); 314 } 315 #endif 316 kunmap_atomic(v); 317 spin_unlock(lock); 318 319 pt->count = 0; 320 pt->pd = pd; 321 pt->index = 0; 322 323 return pt; 324 } 325 326 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 327 unsigned long addr) 328 { 329 uint32_t index = psb_mmu_pd_index(addr); 330 struct psb_mmu_pt *pt; 331 uint32_t *v; 332 spinlock_t *lock = &pd->driver->lock; 333 334 spin_lock(lock); 335 pt = pd->tables[index]; 336 while (!pt) { 337 spin_unlock(lock); 338 pt = psb_mmu_alloc_pt(pd); 339 if (!pt) 340 return NULL; 341 spin_lock(lock); 342 343 if (pd->tables[index]) { 344 spin_unlock(lock); 345 psb_mmu_free_pt(pt); 346 spin_lock(lock); 347 pt = pd->tables[index]; 348 continue; 349 } 350 351 v = kmap_atomic(pd->p); 352 pd->tables[index] = pt; 353 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 354 pt->index = index; 355 kunmap_atomic((void *) v); 356 357 if (pd->hw_context != -1) { 358 psb_mmu_clflush(pd->driver, (void *)&v[index]); 359 atomic_set(&pd->driver->needs_tlbflush, 1); 360 } 361 } 362 pt->v = kmap_atomic(pt->p); 363 return pt; 364 } 365 366 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 367 unsigned long addr) 368 { 369 uint32_t index = psb_mmu_pd_index(addr); 370 struct psb_mmu_pt *pt; 371 spinlock_t *lock = &pd->driver->lock; 372 373 spin_lock(lock); 374 pt = pd->tables[index]; 375 if (!pt) { 376 spin_unlock(lock); 377 return NULL; 378 } 379 pt->v = kmap_atomic(pt->p); 380 return pt; 381 } 382 383 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 384 { 385 struct psb_mmu_pd *pd = pt->pd; 386 uint32_t *v; 387 388 kunmap_atomic(pt->v); 389 if (pt->count == 0) { 390 v = kmap_atomic(pd->p); 391 v[pt->index] = pd->invalid_pde; 392 pd->tables[pt->index] = NULL; 393 394 if (pd->hw_context != -1) { 395 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]); 396 atomic_set(&pd->driver->needs_tlbflush, 1); 397 } 398 kunmap_atomic(v); 399 spin_unlock(&pd->driver->lock); 400 psb_mmu_free_pt(pt); 401 return; 402 } 403 spin_unlock(&pd->driver->lock); 404 } 405 406 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr, 407 uint32_t pte) 408 { 409 pt->v[psb_mmu_pt_index(addr)] = pte; 410 } 411 412 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 413 unsigned long addr) 414 { 415 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 416 } 417 418 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 419 { 420 struct psb_mmu_pd *pd; 421 422 down_read(&driver->sem); 423 pd = driver->default_pd; 424 up_read(&driver->sem); 425 426 return pd; 427 } 428 429 /* Returns the physical address of the PD shared by sgx/msvdx */ 430 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver) 431 { 432 struct psb_mmu_pd *pd; 433 434 pd = psb_mmu_get_default_pd(driver); 435 return page_to_pfn(pd->p) << PAGE_SHIFT; 436 } 437 438 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 439 { 440 struct drm_device *dev = driver->dev; 441 struct drm_psb_private *dev_priv = dev->dev_private; 442 443 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL); 444 psb_mmu_free_pagedir(driver->default_pd); 445 kfree(driver); 446 } 447 448 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev, 449 int trap_pagefaults, 450 int invalid_type, 451 atomic_t *msvdx_mmu_invaldc) 452 { 453 struct psb_mmu_driver *driver; 454 struct drm_psb_private *dev_priv = dev->dev_private; 455 456 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 457 458 if (!driver) 459 return NULL; 460 461 driver->dev = dev; 462 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 463 invalid_type); 464 if (!driver->default_pd) 465 goto out_err1; 466 467 spin_lock_init(&driver->lock); 468 init_rwsem(&driver->sem); 469 down_write(&driver->sem); 470 atomic_set(&driver->needs_tlbflush, 1); 471 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc; 472 473 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL); 474 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT, 475 PSB_CR_BIF_CTRL); 476 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT, 477 PSB_CR_BIF_CTRL); 478 479 driver->has_clflush = 0; 480 481 #if defined(CONFIG_X86) 482 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) { 483 uint32_t tfms, misc, cap0, cap4, clflush_size; 484 485 /* 486 * clflush size is determined at kernel setup for x86_64 but not 487 * for i386. We have to do it here. 488 */ 489 490 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 491 clflush_size = ((misc >> 8) & 0xff) * 8; 492 driver->has_clflush = 1; 493 driver->clflush_add = 494 PAGE_SIZE * clflush_size / sizeof(uint32_t); 495 driver->clflush_mask = driver->clflush_add - 1; 496 driver->clflush_mask = ~driver->clflush_mask; 497 } 498 #endif 499 500 up_write(&driver->sem); 501 return driver; 502 503 out_err1: 504 kfree(driver); 505 return NULL; 506 } 507 508 #if defined(CONFIG_X86) 509 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 510 uint32_t num_pages, uint32_t desired_tile_stride, 511 uint32_t hw_tile_stride) 512 { 513 struct psb_mmu_pt *pt; 514 uint32_t rows = 1; 515 uint32_t i; 516 unsigned long addr; 517 unsigned long end; 518 unsigned long next; 519 unsigned long add; 520 unsigned long row_add; 521 unsigned long clflush_add = pd->driver->clflush_add; 522 unsigned long clflush_mask = pd->driver->clflush_mask; 523 524 if (!pd->driver->has_clflush) 525 return; 526 527 if (hw_tile_stride) 528 rows = num_pages / desired_tile_stride; 529 else 530 desired_tile_stride = num_pages; 531 532 add = desired_tile_stride << PAGE_SHIFT; 533 row_add = hw_tile_stride << PAGE_SHIFT; 534 mb(); 535 for (i = 0; i < rows; ++i) { 536 537 addr = address; 538 end = addr + add; 539 540 do { 541 next = psb_pd_addr_end(addr, end); 542 pt = psb_mmu_pt_map_lock(pd, addr); 543 if (!pt) 544 continue; 545 do { 546 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]); 547 } while (addr += clflush_add, 548 (addr & clflush_mask) < next); 549 550 psb_mmu_pt_unmap_unlock(pt); 551 } while (addr = next, next != end); 552 address += row_add; 553 } 554 mb(); 555 } 556 #else 557 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 558 uint32_t num_pages, uint32_t desired_tile_stride, 559 uint32_t hw_tile_stride) 560 { 561 drm_ttm_cache_flush(); 562 } 563 #endif 564 565 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 566 unsigned long address, uint32_t num_pages) 567 { 568 struct psb_mmu_pt *pt; 569 unsigned long addr; 570 unsigned long end; 571 unsigned long next; 572 unsigned long f_address = address; 573 574 down_read(&pd->driver->sem); 575 576 addr = address; 577 end = addr + (num_pages << PAGE_SHIFT); 578 579 do { 580 next = psb_pd_addr_end(addr, end); 581 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 582 if (!pt) 583 goto out; 584 do { 585 psb_mmu_invalidate_pte(pt, addr); 586 --pt->count; 587 } while (addr += PAGE_SIZE, addr < next); 588 psb_mmu_pt_unmap_unlock(pt); 589 590 } while (addr = next, next != end); 591 592 out: 593 if (pd->hw_context != -1) 594 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 595 596 up_read(&pd->driver->sem); 597 598 if (pd->hw_context != -1) 599 psb_mmu_flush(pd->driver); 600 601 return; 602 } 603 604 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 605 uint32_t num_pages, uint32_t desired_tile_stride, 606 uint32_t hw_tile_stride) 607 { 608 struct psb_mmu_pt *pt; 609 uint32_t rows = 1; 610 uint32_t i; 611 unsigned long addr; 612 unsigned long end; 613 unsigned long next; 614 unsigned long add; 615 unsigned long row_add; 616 unsigned long f_address = address; 617 618 if (hw_tile_stride) 619 rows = num_pages / desired_tile_stride; 620 else 621 desired_tile_stride = num_pages; 622 623 add = desired_tile_stride << PAGE_SHIFT; 624 row_add = hw_tile_stride << PAGE_SHIFT; 625 626 down_read(&pd->driver->sem); 627 628 /* Make sure we only need to flush this processor's cache */ 629 630 for (i = 0; i < rows; ++i) { 631 632 addr = address; 633 end = addr + add; 634 635 do { 636 next = psb_pd_addr_end(addr, end); 637 pt = psb_mmu_pt_map_lock(pd, addr); 638 if (!pt) 639 continue; 640 do { 641 psb_mmu_invalidate_pte(pt, addr); 642 --pt->count; 643 644 } while (addr += PAGE_SIZE, addr < next); 645 psb_mmu_pt_unmap_unlock(pt); 646 647 } while (addr = next, next != end); 648 address += row_add; 649 } 650 if (pd->hw_context != -1) 651 psb_mmu_flush_ptes(pd, f_address, num_pages, 652 desired_tile_stride, hw_tile_stride); 653 654 up_read(&pd->driver->sem); 655 656 if (pd->hw_context != -1) 657 psb_mmu_flush(pd->driver); 658 } 659 660 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 661 unsigned long address, uint32_t num_pages, 662 int type) 663 { 664 struct psb_mmu_pt *pt; 665 uint32_t pte; 666 unsigned long addr; 667 unsigned long end; 668 unsigned long next; 669 unsigned long f_address = address; 670 int ret = -ENOMEM; 671 672 down_read(&pd->driver->sem); 673 674 addr = address; 675 end = addr + (num_pages << PAGE_SHIFT); 676 677 do { 678 next = psb_pd_addr_end(addr, end); 679 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 680 if (!pt) { 681 ret = -ENOMEM; 682 goto out; 683 } 684 do { 685 pte = psb_mmu_mask_pte(start_pfn++, type); 686 psb_mmu_set_pte(pt, addr, pte); 687 pt->count++; 688 } while (addr += PAGE_SIZE, addr < next); 689 psb_mmu_pt_unmap_unlock(pt); 690 691 } while (addr = next, next != end); 692 ret = 0; 693 694 out: 695 if (pd->hw_context != -1) 696 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 697 698 up_read(&pd->driver->sem); 699 700 if (pd->hw_context != -1) 701 psb_mmu_flush(pd->driver); 702 703 return 0; 704 } 705 706 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 707 unsigned long address, uint32_t num_pages, 708 uint32_t desired_tile_stride, uint32_t hw_tile_stride, 709 int type) 710 { 711 struct psb_mmu_pt *pt; 712 uint32_t rows = 1; 713 uint32_t i; 714 uint32_t pte; 715 unsigned long addr; 716 unsigned long end; 717 unsigned long next; 718 unsigned long add; 719 unsigned long row_add; 720 unsigned long f_address = address; 721 int ret = -ENOMEM; 722 723 if (hw_tile_stride) { 724 if (num_pages % desired_tile_stride != 0) 725 return -EINVAL; 726 rows = num_pages / desired_tile_stride; 727 } else { 728 desired_tile_stride = num_pages; 729 } 730 731 add = desired_tile_stride << PAGE_SHIFT; 732 row_add = hw_tile_stride << PAGE_SHIFT; 733 734 down_read(&pd->driver->sem); 735 736 for (i = 0; i < rows; ++i) { 737 738 addr = address; 739 end = addr + add; 740 741 do { 742 next = psb_pd_addr_end(addr, end); 743 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 744 if (!pt) 745 goto out; 746 do { 747 pte = psb_mmu_mask_pte(page_to_pfn(*pages++), 748 type); 749 psb_mmu_set_pte(pt, addr, pte); 750 pt->count++; 751 } while (addr += PAGE_SIZE, addr < next); 752 psb_mmu_pt_unmap_unlock(pt); 753 754 } while (addr = next, next != end); 755 756 address += row_add; 757 } 758 759 ret = 0; 760 out: 761 if (pd->hw_context != -1) 762 psb_mmu_flush_ptes(pd, f_address, num_pages, 763 desired_tile_stride, hw_tile_stride); 764 765 up_read(&pd->driver->sem); 766 767 if (pd->hw_context != -1) 768 psb_mmu_flush(pd->driver); 769 770 return ret; 771 } 772 773 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 774 unsigned long *pfn) 775 { 776 int ret; 777 struct psb_mmu_pt *pt; 778 uint32_t tmp; 779 spinlock_t *lock = &pd->driver->lock; 780 781 down_read(&pd->driver->sem); 782 pt = psb_mmu_pt_map_lock(pd, virtual); 783 if (!pt) { 784 uint32_t *v; 785 786 spin_lock(lock); 787 v = kmap_atomic(pd->p); 788 tmp = v[psb_mmu_pd_index(virtual)]; 789 kunmap_atomic(v); 790 spin_unlock(lock); 791 792 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 793 !(pd->invalid_pte & PSB_PTE_VALID)) { 794 ret = -EINVAL; 795 goto out; 796 } 797 ret = 0; 798 *pfn = pd->invalid_pte >> PAGE_SHIFT; 799 goto out; 800 } 801 tmp = pt->v[psb_mmu_pt_index(virtual)]; 802 if (!(tmp & PSB_PTE_VALID)) { 803 ret = -EINVAL; 804 } else { 805 ret = 0; 806 *pfn = tmp >> PAGE_SHIFT; 807 } 808 psb_mmu_pt_unmap_unlock(pt); 809 out: 810 up_read(&pd->driver->sem); 811 return ret; 812 } 813