1 /* 2 * Copyright © 2010 Daniel Vetter 3 * Copyright © 2011-2014 Intel Corporation 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/slab.h> /* fault-inject.h is not standalone! */ 27 28 #include <linux/fault-inject.h> 29 #include <linux/log2.h> 30 #include <linux/random.h> 31 #include <linux/seq_file.h> 32 #include <linux/stop_machine.h> 33 34 #include <asm/set_memory.h> 35 36 #include <drm/drmP.h> 37 #include <drm/i915_drm.h> 38 39 #include "i915_drv.h" 40 #include "i915_vgpu.h" 41 #include "i915_trace.h" 42 #include "intel_drv.h" 43 #include "intel_frontbuffer.h" 44 45 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM) 46 47 /** 48 * DOC: Global GTT views 49 * 50 * Background and previous state 51 * 52 * Historically objects could exists (be bound) in global GTT space only as 53 * singular instances with a view representing all of the object's backing pages 54 * in a linear fashion. This view will be called a normal view. 55 * 56 * To support multiple views of the same object, where the number of mapped 57 * pages is not equal to the backing store, or where the layout of the pages 58 * is not linear, concept of a GGTT view was added. 59 * 60 * One example of an alternative view is a stereo display driven by a single 61 * image. In this case we would have a framebuffer looking like this 62 * (2x2 pages): 63 * 64 * 12 65 * 34 66 * 67 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 68 * rendering. In contrast, fed to the display engine would be an alternative 69 * view which could look something like this: 70 * 71 * 1212 72 * 3434 73 * 74 * In this example both the size and layout of pages in the alternative view is 75 * different from the normal view. 76 * 77 * Implementation and usage 78 * 79 * GGTT views are implemented using VMAs and are distinguished via enum 80 * i915_ggtt_view_type and struct i915_ggtt_view. 81 * 82 * A new flavour of core GEM functions which work with GGTT bound objects were 83 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 84 * renaming in large amounts of code. They take the struct i915_ggtt_view 85 * parameter encapsulating all metadata required to implement a view. 86 * 87 * As a helper for callers which are only interested in the normal view, 88 * globally const i915_ggtt_view_normal singleton instance exists. All old core 89 * GEM API functions, the ones not taking the view parameter, are operating on, 90 * or with the normal GGTT view. 91 * 92 * Code wanting to add or use a new GGTT view needs to: 93 * 94 * 1. Add a new enum with a suitable name. 95 * 2. Extend the metadata in the i915_ggtt_view structure if required. 96 * 3. Add support to i915_get_vma_pages(). 97 * 98 * New views are required to build a scatter-gather table from within the 99 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 100 * exists for the lifetime of an VMA. 101 * 102 * Core API is designed to have copy semantics which means that passed in 103 * struct i915_ggtt_view does not need to be persistent (left around after 104 * calling the core API functions). 105 * 106 */ 107 108 static int 109 i915_get_ggtt_vma_pages(struct i915_vma *vma); 110 111 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv) 112 { 113 /* Note that as an uncached mmio write, this should flush the 114 * WCB of the writes into the GGTT before it triggers the invalidate. 115 */ 116 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 117 } 118 119 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv) 120 { 121 gen6_ggtt_invalidate(dev_priv); 122 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); 123 } 124 125 static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv) 126 { 127 intel_gtt_chipset_flush(); 128 } 129 130 static inline void i915_ggtt_invalidate(struct drm_i915_private *i915) 131 { 132 i915->ggtt.invalidate(i915); 133 } 134 135 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, 136 int enable_ppgtt) 137 { 138 bool has_aliasing_ppgtt; 139 bool has_full_ppgtt; 140 bool has_full_48bit_ppgtt; 141 142 has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt; 143 has_full_ppgtt = dev_priv->info.has_full_ppgtt; 144 has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt; 145 146 if (intel_vgpu_active(dev_priv)) { 147 /* emulation is too hard */ 148 has_full_ppgtt = false; 149 has_full_48bit_ppgtt = false; 150 } 151 152 if (!has_aliasing_ppgtt) 153 return 0; 154 155 /* 156 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 157 * execlists, the sole mechanism available to submit work. 158 */ 159 if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9) 160 return 0; 161 162 if (enable_ppgtt == 1) 163 return 1; 164 165 if (enable_ppgtt == 2 && has_full_ppgtt) 166 return 2; 167 168 if (enable_ppgtt == 3 && has_full_48bit_ppgtt) 169 return 3; 170 171 #ifdef CONFIG_INTEL_IOMMU 172 /* Disable ppgtt on SNB if VT-d is on. */ 173 if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) { 174 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 175 return 0; 176 } 177 #endif 178 179 /* Early VLV doesn't have this */ 180 if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) { 181 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 182 return 0; 183 } 184 185 if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt) 186 return has_full_48bit_ppgtt ? 3 : 2; 187 else 188 return has_aliasing_ppgtt ? 1 : 0; 189 } 190 191 static int ppgtt_bind_vma(struct i915_vma *vma, 192 enum i915_cache_level cache_level, 193 u32 unused) 194 { 195 u32 pte_flags; 196 int ret; 197 198 if (!(vma->flags & I915_VMA_LOCAL_BIND)) { 199 ret = vma->vm->allocate_va_range(vma->vm, vma->node.start, 200 vma->size); 201 if (ret) 202 return ret; 203 } 204 205 vma->pages = vma->obj->mm.pages; 206 207 /* Currently applicable only to VLV */ 208 pte_flags = 0; 209 if (vma->obj->gt_ro) 210 pte_flags |= PTE_READ_ONLY; 211 212 vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, 213 cache_level, pte_flags); 214 215 return 0; 216 } 217 218 static void ppgtt_unbind_vma(struct i915_vma *vma) 219 { 220 vma->vm->clear_range(vma->vm, vma->node.start, vma->size); 221 } 222 223 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 224 enum i915_cache_level level) 225 { 226 gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW; 227 pte |= addr; 228 229 switch (level) { 230 case I915_CACHE_NONE: 231 pte |= PPAT_UNCACHED_INDEX; 232 break; 233 case I915_CACHE_WT: 234 pte |= PPAT_DISPLAY_ELLC_INDEX; 235 break; 236 default: 237 pte |= PPAT_CACHED_INDEX; 238 break; 239 } 240 241 return pte; 242 } 243 244 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 245 const enum i915_cache_level level) 246 { 247 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 248 pde |= addr; 249 if (level != I915_CACHE_NONE) 250 pde |= PPAT_CACHED_PDE_INDEX; 251 else 252 pde |= PPAT_UNCACHED_INDEX; 253 return pde; 254 } 255 256 #define gen8_pdpe_encode gen8_pde_encode 257 #define gen8_pml4e_encode gen8_pde_encode 258 259 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 260 enum i915_cache_level level, 261 u32 unused) 262 { 263 gen6_pte_t pte = GEN6_PTE_VALID; 264 pte |= GEN6_PTE_ADDR_ENCODE(addr); 265 266 switch (level) { 267 case I915_CACHE_L3_LLC: 268 case I915_CACHE_LLC: 269 pte |= GEN6_PTE_CACHE_LLC; 270 break; 271 case I915_CACHE_NONE: 272 pte |= GEN6_PTE_UNCACHED; 273 break; 274 default: 275 MISSING_CASE(level); 276 } 277 278 return pte; 279 } 280 281 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 282 enum i915_cache_level level, 283 u32 unused) 284 { 285 gen6_pte_t pte = GEN6_PTE_VALID; 286 pte |= GEN6_PTE_ADDR_ENCODE(addr); 287 288 switch (level) { 289 case I915_CACHE_L3_LLC: 290 pte |= GEN7_PTE_CACHE_L3_LLC; 291 break; 292 case I915_CACHE_LLC: 293 pte |= GEN6_PTE_CACHE_LLC; 294 break; 295 case I915_CACHE_NONE: 296 pte |= GEN6_PTE_UNCACHED; 297 break; 298 default: 299 MISSING_CASE(level); 300 } 301 302 return pte; 303 } 304 305 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 306 enum i915_cache_level level, 307 u32 flags) 308 { 309 gen6_pte_t pte = GEN6_PTE_VALID; 310 pte |= GEN6_PTE_ADDR_ENCODE(addr); 311 312 if (!(flags & PTE_READ_ONLY)) 313 pte |= BYT_PTE_WRITEABLE; 314 315 if (level != I915_CACHE_NONE) 316 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 317 318 return pte; 319 } 320 321 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 322 enum i915_cache_level level, 323 u32 unused) 324 { 325 gen6_pte_t pte = GEN6_PTE_VALID; 326 pte |= HSW_PTE_ADDR_ENCODE(addr); 327 328 if (level != I915_CACHE_NONE) 329 pte |= HSW_WB_LLC_AGE3; 330 331 return pte; 332 } 333 334 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 335 enum i915_cache_level level, 336 u32 unused) 337 { 338 gen6_pte_t pte = GEN6_PTE_VALID; 339 pte |= HSW_PTE_ADDR_ENCODE(addr); 340 341 switch (level) { 342 case I915_CACHE_NONE: 343 break; 344 case I915_CACHE_WT: 345 pte |= HSW_WT_ELLC_LLC_AGE3; 346 break; 347 default: 348 pte |= HSW_WB_ELLC_LLC_AGE3; 349 break; 350 } 351 352 return pte; 353 } 354 355 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp) 356 { 357 struct page *page; 358 359 if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1))) 360 i915_gem_shrink_all(vm->i915); 361 362 if (vm->free_pages.nr) 363 return vm->free_pages.pages[--vm->free_pages.nr]; 364 365 page = alloc_page(gfp); 366 if (!page) 367 return NULL; 368 369 if (vm->pt_kmap_wc) 370 set_pages_array_wc(&page, 1); 371 372 return page; 373 } 374 375 static void vm_free_pages_release(struct i915_address_space *vm) 376 { 377 GEM_BUG_ON(!pagevec_count(&vm->free_pages)); 378 379 if (vm->pt_kmap_wc) 380 set_pages_array_wb(vm->free_pages.pages, 381 pagevec_count(&vm->free_pages)); 382 383 __pagevec_release(&vm->free_pages); 384 } 385 386 static void vm_free_page(struct i915_address_space *vm, struct page *page) 387 { 388 if (!pagevec_add(&vm->free_pages, page)) 389 vm_free_pages_release(vm); 390 } 391 392 static int __setup_page_dma(struct i915_address_space *vm, 393 struct i915_page_dma *p, 394 gfp_t gfp) 395 { 396 p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY); 397 if (unlikely(!p->page)) 398 return -ENOMEM; 399 400 p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE, 401 PCI_DMA_BIDIRECTIONAL); 402 if (unlikely(dma_mapping_error(vm->dma, p->daddr))) { 403 vm_free_page(vm, p->page); 404 return -ENOMEM; 405 } 406 407 return 0; 408 } 409 410 static int setup_page_dma(struct i915_address_space *vm, 411 struct i915_page_dma *p) 412 { 413 return __setup_page_dma(vm, p, I915_GFP_DMA); 414 } 415 416 static void cleanup_page_dma(struct i915_address_space *vm, 417 struct i915_page_dma *p) 418 { 419 dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 420 vm_free_page(vm, p->page); 421 } 422 423 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page) 424 425 #define setup_px(vm, px) setup_page_dma((vm), px_base(px)) 426 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px)) 427 #define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v)) 428 #define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v)) 429 430 static void fill_page_dma(struct i915_address_space *vm, 431 struct i915_page_dma *p, 432 const u64 val) 433 { 434 u64 * const vaddr = kmap_atomic(p->page); 435 int i; 436 437 for (i = 0; i < 512; i++) 438 vaddr[i] = val; 439 440 kunmap_atomic(vaddr); 441 } 442 443 static void fill_page_dma_32(struct i915_address_space *vm, 444 struct i915_page_dma *p, 445 const u32 v) 446 { 447 fill_page_dma(vm, p, (u64)v << 32 | v); 448 } 449 450 static int 451 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) 452 { 453 return __setup_page_dma(vm, &vm->scratch_page, gfp | __GFP_ZERO); 454 } 455 456 static void cleanup_scratch_page(struct i915_address_space *vm) 457 { 458 cleanup_page_dma(vm, &vm->scratch_page); 459 } 460 461 static struct i915_page_table *alloc_pt(struct i915_address_space *vm) 462 { 463 struct i915_page_table *pt; 464 465 pt = kmalloc(sizeof(*pt), GFP_KERNEL | __GFP_NOWARN); 466 if (unlikely(!pt)) 467 return ERR_PTR(-ENOMEM); 468 469 if (unlikely(setup_px(vm, pt))) { 470 kfree(pt); 471 return ERR_PTR(-ENOMEM); 472 } 473 474 pt->used_ptes = 0; 475 return pt; 476 } 477 478 static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) 479 { 480 cleanup_px(vm, pt); 481 kfree(pt); 482 } 483 484 static void gen8_initialize_pt(struct i915_address_space *vm, 485 struct i915_page_table *pt) 486 { 487 fill_px(vm, pt, 488 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC)); 489 } 490 491 static void gen6_initialize_pt(struct i915_address_space *vm, 492 struct i915_page_table *pt) 493 { 494 fill32_px(vm, pt, 495 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0)); 496 } 497 498 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) 499 { 500 struct i915_page_directory *pd; 501 502 pd = kzalloc(sizeof(*pd), GFP_KERNEL | __GFP_NOWARN); 503 if (unlikely(!pd)) 504 return ERR_PTR(-ENOMEM); 505 506 if (unlikely(setup_px(vm, pd))) { 507 kfree(pd); 508 return ERR_PTR(-ENOMEM); 509 } 510 511 pd->used_pdes = 0; 512 return pd; 513 } 514 515 static void free_pd(struct i915_address_space *vm, 516 struct i915_page_directory *pd) 517 { 518 cleanup_px(vm, pd); 519 kfree(pd); 520 } 521 522 static void gen8_initialize_pd(struct i915_address_space *vm, 523 struct i915_page_directory *pd) 524 { 525 unsigned int i; 526 527 fill_px(vm, pd, 528 gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); 529 for (i = 0; i < I915_PDES; i++) 530 pd->page_table[i] = vm->scratch_pt; 531 } 532 533 static int __pdp_init(struct i915_address_space *vm, 534 struct i915_page_directory_pointer *pdp) 535 { 536 const unsigned int pdpes = i915_pdpes_per_pdp(vm); 537 unsigned int i; 538 539 pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory), 540 GFP_KERNEL | __GFP_NOWARN); 541 if (unlikely(!pdp->page_directory)) 542 return -ENOMEM; 543 544 for (i = 0; i < pdpes; i++) 545 pdp->page_directory[i] = vm->scratch_pd; 546 547 return 0; 548 } 549 550 static void __pdp_fini(struct i915_page_directory_pointer *pdp) 551 { 552 kfree(pdp->page_directory); 553 pdp->page_directory = NULL; 554 } 555 556 static inline bool use_4lvl(const struct i915_address_space *vm) 557 { 558 return i915_vm_is_48bit(vm); 559 } 560 561 static struct i915_page_directory_pointer * 562 alloc_pdp(struct i915_address_space *vm) 563 { 564 struct i915_page_directory_pointer *pdp; 565 int ret = -ENOMEM; 566 567 WARN_ON(!use_4lvl(vm)); 568 569 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 570 if (!pdp) 571 return ERR_PTR(-ENOMEM); 572 573 ret = __pdp_init(vm, pdp); 574 if (ret) 575 goto fail_bitmap; 576 577 ret = setup_px(vm, pdp); 578 if (ret) 579 goto fail_page_m; 580 581 return pdp; 582 583 fail_page_m: 584 __pdp_fini(pdp); 585 fail_bitmap: 586 kfree(pdp); 587 588 return ERR_PTR(ret); 589 } 590 591 static void free_pdp(struct i915_address_space *vm, 592 struct i915_page_directory_pointer *pdp) 593 { 594 __pdp_fini(pdp); 595 596 if (!use_4lvl(vm)) 597 return; 598 599 cleanup_px(vm, pdp); 600 kfree(pdp); 601 } 602 603 static void gen8_initialize_pdp(struct i915_address_space *vm, 604 struct i915_page_directory_pointer *pdp) 605 { 606 gen8_ppgtt_pdpe_t scratch_pdpe; 607 608 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); 609 610 fill_px(vm, pdp, scratch_pdpe); 611 } 612 613 static void gen8_initialize_pml4(struct i915_address_space *vm, 614 struct i915_pml4 *pml4) 615 { 616 unsigned int i; 617 618 fill_px(vm, pml4, 619 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); 620 for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) 621 pml4->pdps[i] = vm->scratch_pdp; 622 } 623 624 /* Broadwell Page Directory Pointer Descriptors */ 625 static int gen8_write_pdp(struct drm_i915_gem_request *req, 626 unsigned entry, 627 dma_addr_t addr) 628 { 629 struct intel_engine_cs *engine = req->engine; 630 u32 *cs; 631 632 BUG_ON(entry >= 4); 633 634 cs = intel_ring_begin(req, 6); 635 if (IS_ERR(cs)) 636 return PTR_ERR(cs); 637 638 *cs++ = MI_LOAD_REGISTER_IMM(1); 639 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry)); 640 *cs++ = upper_32_bits(addr); 641 *cs++ = MI_LOAD_REGISTER_IMM(1); 642 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry)); 643 *cs++ = lower_32_bits(addr); 644 intel_ring_advance(req, cs); 645 646 return 0; 647 } 648 649 static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt, 650 struct drm_i915_gem_request *req) 651 { 652 int i, ret; 653 654 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { 655 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 656 657 ret = gen8_write_pdp(req, i, pd_daddr); 658 if (ret) 659 return ret; 660 } 661 662 return 0; 663 } 664 665 static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt, 666 struct drm_i915_gem_request *req) 667 { 668 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 669 } 670 671 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 672 * the page table structures, we mark them dirty so that 673 * context switching/execlist queuing code takes extra steps 674 * to ensure that tlbs are flushed. 675 */ 676 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 677 { 678 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask; 679 } 680 681 /* Removes entries from a single page table, releasing it if it's empty. 682 * Caller can use the return value to update higher-level entries. 683 */ 684 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, 685 struct i915_page_table *pt, 686 u64 start, u64 length) 687 { 688 unsigned int num_entries = gen8_pte_count(start, length); 689 unsigned int pte = gen8_pte_index(start); 690 unsigned int pte_end = pte + num_entries; 691 const gen8_pte_t scratch_pte = 692 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); 693 gen8_pte_t *vaddr; 694 695 GEM_BUG_ON(num_entries > pt->used_ptes); 696 697 pt->used_ptes -= num_entries; 698 if (!pt->used_ptes) 699 return true; 700 701 vaddr = kmap_atomic_px(pt); 702 while (pte < pte_end) 703 vaddr[pte++] = scratch_pte; 704 kunmap_atomic(vaddr); 705 706 return false; 707 } 708 709 static void gen8_ppgtt_set_pde(struct i915_address_space *vm, 710 struct i915_page_directory *pd, 711 struct i915_page_table *pt, 712 unsigned int pde) 713 { 714 gen8_pde_t *vaddr; 715 716 pd->page_table[pde] = pt; 717 718 vaddr = kmap_atomic_px(pd); 719 vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC); 720 kunmap_atomic(vaddr); 721 } 722 723 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, 724 struct i915_page_directory *pd, 725 u64 start, u64 length) 726 { 727 struct i915_page_table *pt; 728 u32 pde; 729 730 gen8_for_each_pde(pt, pd, start, length, pde) { 731 GEM_BUG_ON(pt == vm->scratch_pt); 732 733 if (!gen8_ppgtt_clear_pt(vm, pt, start, length)) 734 continue; 735 736 gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); 737 GEM_BUG_ON(!pd->used_pdes); 738 pd->used_pdes--; 739 740 free_pt(vm, pt); 741 } 742 743 return !pd->used_pdes; 744 } 745 746 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, 747 struct i915_page_directory_pointer *pdp, 748 struct i915_page_directory *pd, 749 unsigned int pdpe) 750 { 751 gen8_ppgtt_pdpe_t *vaddr; 752 753 pdp->page_directory[pdpe] = pd; 754 if (!use_4lvl(vm)) 755 return; 756 757 vaddr = kmap_atomic_px(pdp); 758 vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); 759 kunmap_atomic(vaddr); 760 } 761 762 /* Removes entries from a single page dir pointer, releasing it if it's empty. 763 * Caller can use the return value to update higher-level entries 764 */ 765 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, 766 struct i915_page_directory_pointer *pdp, 767 u64 start, u64 length) 768 { 769 struct i915_page_directory *pd; 770 unsigned int pdpe; 771 772 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 773 GEM_BUG_ON(pd == vm->scratch_pd); 774 775 if (!gen8_ppgtt_clear_pd(vm, pd, start, length)) 776 continue; 777 778 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); 779 GEM_BUG_ON(!pdp->used_pdpes); 780 pdp->used_pdpes--; 781 782 free_pd(vm, pd); 783 } 784 785 return !pdp->used_pdpes; 786 } 787 788 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, 789 u64 start, u64 length) 790 { 791 gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length); 792 } 793 794 static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4, 795 struct i915_page_directory_pointer *pdp, 796 unsigned int pml4e) 797 { 798 gen8_ppgtt_pml4e_t *vaddr; 799 800 pml4->pdps[pml4e] = pdp; 801 802 vaddr = kmap_atomic_px(pml4); 803 vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 804 kunmap_atomic(vaddr); 805 } 806 807 /* Removes entries from a single pml4. 808 * This is the top-level structure in 4-level page tables used on gen8+. 809 * Empty entries are always scratch pml4e. 810 */ 811 static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, 812 u64 start, u64 length) 813 { 814 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 815 struct i915_pml4 *pml4 = &ppgtt->pml4; 816 struct i915_page_directory_pointer *pdp; 817 unsigned int pml4e; 818 819 GEM_BUG_ON(!use_4lvl(vm)); 820 821 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 822 GEM_BUG_ON(pdp == vm->scratch_pdp); 823 824 if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length)) 825 continue; 826 827 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); 828 829 free_pdp(vm, pdp); 830 } 831 } 832 833 struct sgt_dma { 834 struct scatterlist *sg; 835 dma_addr_t dma, max; 836 }; 837 838 struct gen8_insert_pte { 839 u16 pml4e; 840 u16 pdpe; 841 u16 pde; 842 u16 pte; 843 }; 844 845 static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start) 846 { 847 return (struct gen8_insert_pte) { 848 gen8_pml4e_index(start), 849 gen8_pdpe_index(start), 850 gen8_pde_index(start), 851 gen8_pte_index(start), 852 }; 853 } 854 855 static __always_inline bool 856 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt, 857 struct i915_page_directory_pointer *pdp, 858 struct sgt_dma *iter, 859 struct gen8_insert_pte *idx, 860 enum i915_cache_level cache_level) 861 { 862 struct i915_page_directory *pd; 863 const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level); 864 gen8_pte_t *vaddr; 865 bool ret; 866 867 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base)); 868 pd = pdp->page_directory[idx->pdpe]; 869 vaddr = kmap_atomic_px(pd->page_table[idx->pde]); 870 do { 871 vaddr[idx->pte] = pte_encode | iter->dma; 872 873 iter->dma += PAGE_SIZE; 874 if (iter->dma >= iter->max) { 875 iter->sg = __sg_next(iter->sg); 876 if (!iter->sg) { 877 ret = false; 878 break; 879 } 880 881 iter->dma = sg_dma_address(iter->sg); 882 iter->max = iter->dma + iter->sg->length; 883 } 884 885 if (++idx->pte == GEN8_PTES) { 886 idx->pte = 0; 887 888 if (++idx->pde == I915_PDES) { 889 idx->pde = 0; 890 891 /* Limited by sg length for 3lvl */ 892 if (++idx->pdpe == GEN8_PML4ES_PER_PML4) { 893 idx->pdpe = 0; 894 ret = true; 895 break; 896 } 897 898 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base)); 899 pd = pdp->page_directory[idx->pdpe]; 900 } 901 902 kunmap_atomic(vaddr); 903 vaddr = kmap_atomic_px(pd->page_table[idx->pde]); 904 } 905 } while (1); 906 kunmap_atomic(vaddr); 907 908 return ret; 909 } 910 911 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm, 912 struct sg_table *pages, 913 u64 start, 914 enum i915_cache_level cache_level, 915 u32 unused) 916 { 917 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 918 struct sgt_dma iter = { 919 .sg = pages->sgl, 920 .dma = sg_dma_address(iter.sg), 921 .max = iter.dma + iter.sg->length, 922 }; 923 struct gen8_insert_pte idx = gen8_insert_pte(start); 924 925 gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx, 926 cache_level); 927 } 928 929 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, 930 struct sg_table *pages, 931 u64 start, 932 enum i915_cache_level cache_level, 933 u32 unused) 934 { 935 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 936 struct sgt_dma iter = { 937 .sg = pages->sgl, 938 .dma = sg_dma_address(iter.sg), 939 .max = iter.dma + iter.sg->length, 940 }; 941 struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps; 942 struct gen8_insert_pte idx = gen8_insert_pte(start); 943 944 while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], &iter, 945 &idx, cache_level)) 946 GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4); 947 } 948 949 static void gen8_free_page_tables(struct i915_address_space *vm, 950 struct i915_page_directory *pd) 951 { 952 int i; 953 954 if (!px_page(pd)) 955 return; 956 957 for (i = 0; i < I915_PDES; i++) { 958 if (pd->page_table[i] != vm->scratch_pt) 959 free_pt(vm, pd->page_table[i]); 960 } 961 } 962 963 static int gen8_init_scratch(struct i915_address_space *vm) 964 { 965 int ret; 966 967 ret = setup_scratch_page(vm, I915_GFP_DMA); 968 if (ret) 969 return ret; 970 971 vm->scratch_pt = alloc_pt(vm); 972 if (IS_ERR(vm->scratch_pt)) { 973 ret = PTR_ERR(vm->scratch_pt); 974 goto free_scratch_page; 975 } 976 977 vm->scratch_pd = alloc_pd(vm); 978 if (IS_ERR(vm->scratch_pd)) { 979 ret = PTR_ERR(vm->scratch_pd); 980 goto free_pt; 981 } 982 983 if (use_4lvl(vm)) { 984 vm->scratch_pdp = alloc_pdp(vm); 985 if (IS_ERR(vm->scratch_pdp)) { 986 ret = PTR_ERR(vm->scratch_pdp); 987 goto free_pd; 988 } 989 } 990 991 gen8_initialize_pt(vm, vm->scratch_pt); 992 gen8_initialize_pd(vm, vm->scratch_pd); 993 if (use_4lvl(vm)) 994 gen8_initialize_pdp(vm, vm->scratch_pdp); 995 996 return 0; 997 998 free_pd: 999 free_pd(vm, vm->scratch_pd); 1000 free_pt: 1001 free_pt(vm, vm->scratch_pt); 1002 free_scratch_page: 1003 cleanup_scratch_page(vm); 1004 1005 return ret; 1006 } 1007 1008 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) 1009 { 1010 struct i915_address_space *vm = &ppgtt->base; 1011 struct drm_i915_private *dev_priv = vm->i915; 1012 enum vgt_g2v_type msg; 1013 int i; 1014 1015 if (use_4lvl(vm)) { 1016 const u64 daddr = px_dma(&ppgtt->pml4); 1017 1018 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); 1019 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); 1020 1021 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : 1022 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); 1023 } else { 1024 for (i = 0; i < GEN8_3LVL_PDPES; i++) { 1025 const u64 daddr = i915_page_dir_dma_addr(ppgtt, i); 1026 1027 I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr)); 1028 I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr)); 1029 } 1030 1031 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : 1032 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); 1033 } 1034 1035 I915_WRITE(vgtif_reg(g2v_notify), msg); 1036 1037 return 0; 1038 } 1039 1040 static void gen8_free_scratch(struct i915_address_space *vm) 1041 { 1042 if (use_4lvl(vm)) 1043 free_pdp(vm, vm->scratch_pdp); 1044 free_pd(vm, vm->scratch_pd); 1045 free_pt(vm, vm->scratch_pt); 1046 cleanup_scratch_page(vm); 1047 } 1048 1049 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, 1050 struct i915_page_directory_pointer *pdp) 1051 { 1052 const unsigned int pdpes = i915_pdpes_per_pdp(vm); 1053 int i; 1054 1055 for (i = 0; i < pdpes; i++) { 1056 if (pdp->page_directory[i] == vm->scratch_pd) 1057 continue; 1058 1059 gen8_free_page_tables(vm, pdp->page_directory[i]); 1060 free_pd(vm, pdp->page_directory[i]); 1061 } 1062 1063 free_pdp(vm, pdp); 1064 } 1065 1066 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 1067 { 1068 int i; 1069 1070 for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) { 1071 if (ppgtt->pml4.pdps[i] == ppgtt->base.scratch_pdp) 1072 continue; 1073 1074 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]); 1075 } 1076 1077 cleanup_px(&ppgtt->base, &ppgtt->pml4); 1078 } 1079 1080 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 1081 { 1082 struct drm_i915_private *dev_priv = vm->i915; 1083 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1084 1085 if (intel_vgpu_active(dev_priv)) 1086 gen8_ppgtt_notify_vgt(ppgtt, false); 1087 1088 if (use_4lvl(vm)) 1089 gen8_ppgtt_cleanup_4lvl(ppgtt); 1090 else 1091 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp); 1092 1093 gen8_free_scratch(vm); 1094 } 1095 1096 static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, 1097 struct i915_page_directory *pd, 1098 u64 start, u64 length) 1099 { 1100 struct i915_page_table *pt; 1101 u64 from = start; 1102 unsigned int pde; 1103 1104 gen8_for_each_pde(pt, pd, start, length, pde) { 1105 if (pt == vm->scratch_pt) { 1106 pt = alloc_pt(vm); 1107 if (IS_ERR(pt)) 1108 goto unwind; 1109 1110 gen8_initialize_pt(vm, pt); 1111 1112 gen8_ppgtt_set_pde(vm, pd, pt, pde); 1113 pd->used_pdes++; 1114 GEM_BUG_ON(pd->used_pdes > I915_PDES); 1115 } 1116 1117 pt->used_ptes += gen8_pte_count(start, length); 1118 } 1119 return 0; 1120 1121 unwind: 1122 gen8_ppgtt_clear_pd(vm, pd, from, start - from); 1123 return -ENOMEM; 1124 } 1125 1126 static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, 1127 struct i915_page_directory_pointer *pdp, 1128 u64 start, u64 length) 1129 { 1130 struct i915_page_directory *pd; 1131 u64 from = start; 1132 unsigned int pdpe; 1133 int ret; 1134 1135 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1136 if (pd == vm->scratch_pd) { 1137 pd = alloc_pd(vm); 1138 if (IS_ERR(pd)) 1139 goto unwind; 1140 1141 gen8_initialize_pd(vm, pd); 1142 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); 1143 pdp->used_pdpes++; 1144 GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm)); 1145 1146 mark_tlbs_dirty(i915_vm_to_ppgtt(vm)); 1147 } 1148 1149 ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); 1150 if (unlikely(ret)) 1151 goto unwind_pd; 1152 } 1153 1154 return 0; 1155 1156 unwind_pd: 1157 if (!pd->used_pdes) { 1158 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); 1159 GEM_BUG_ON(!pdp->used_pdpes); 1160 pdp->used_pdpes--; 1161 free_pd(vm, pd); 1162 } 1163 unwind: 1164 gen8_ppgtt_clear_pdp(vm, pdp, from, start - from); 1165 return -ENOMEM; 1166 } 1167 1168 static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, 1169 u64 start, u64 length) 1170 { 1171 return gen8_ppgtt_alloc_pdp(vm, 1172 &i915_vm_to_ppgtt(vm)->pdp, start, length); 1173 } 1174 1175 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, 1176 u64 start, u64 length) 1177 { 1178 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1179 struct i915_pml4 *pml4 = &ppgtt->pml4; 1180 struct i915_page_directory_pointer *pdp; 1181 u64 from = start; 1182 u32 pml4e; 1183 int ret; 1184 1185 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1186 if (pml4->pdps[pml4e] == vm->scratch_pdp) { 1187 pdp = alloc_pdp(vm); 1188 if (IS_ERR(pdp)) 1189 goto unwind; 1190 1191 gen8_initialize_pdp(vm, pdp); 1192 gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); 1193 } 1194 1195 ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); 1196 if (unlikely(ret)) 1197 goto unwind_pdp; 1198 } 1199 1200 return 0; 1201 1202 unwind_pdp: 1203 if (!pdp->used_pdpes) { 1204 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); 1205 free_pdp(vm, pdp); 1206 } 1207 unwind: 1208 gen8_ppgtt_clear_4lvl(vm, from, start - from); 1209 return -ENOMEM; 1210 } 1211 1212 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt, 1213 struct i915_page_directory_pointer *pdp, 1214 u64 start, u64 length, 1215 gen8_pte_t scratch_pte, 1216 struct seq_file *m) 1217 { 1218 struct i915_address_space *vm = &ppgtt->base; 1219 struct i915_page_directory *pd; 1220 u32 pdpe; 1221 1222 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1223 struct i915_page_table *pt; 1224 u64 pd_len = length; 1225 u64 pd_start = start; 1226 u32 pde; 1227 1228 if (pdp->page_directory[pdpe] == ppgtt->base.scratch_pd) 1229 continue; 1230 1231 seq_printf(m, "\tPDPE #%d\n", pdpe); 1232 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) { 1233 u32 pte; 1234 gen8_pte_t *pt_vaddr; 1235 1236 if (pd->page_table[pde] == ppgtt->base.scratch_pt) 1237 continue; 1238 1239 pt_vaddr = kmap_atomic_px(pt); 1240 for (pte = 0; pte < GEN8_PTES; pte += 4) { 1241 u64 va = (pdpe << GEN8_PDPE_SHIFT | 1242 pde << GEN8_PDE_SHIFT | 1243 pte << GEN8_PTE_SHIFT); 1244 int i; 1245 bool found = false; 1246 1247 for (i = 0; i < 4; i++) 1248 if (pt_vaddr[pte + i] != scratch_pte) 1249 found = true; 1250 if (!found) 1251 continue; 1252 1253 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); 1254 for (i = 0; i < 4; i++) { 1255 if (pt_vaddr[pte + i] != scratch_pte) 1256 seq_printf(m, " %llx", pt_vaddr[pte + i]); 1257 else 1258 seq_puts(m, " SCRATCH "); 1259 } 1260 seq_puts(m, "\n"); 1261 } 1262 kunmap_atomic(pt_vaddr); 1263 } 1264 } 1265 } 1266 1267 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1268 { 1269 struct i915_address_space *vm = &ppgtt->base; 1270 const gen8_pte_t scratch_pte = 1271 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); 1272 u64 start = 0, length = ppgtt->base.total; 1273 1274 if (use_4lvl(vm)) { 1275 u64 pml4e; 1276 struct i915_pml4 *pml4 = &ppgtt->pml4; 1277 struct i915_page_directory_pointer *pdp; 1278 1279 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1280 if (pml4->pdps[pml4e] == ppgtt->base.scratch_pdp) 1281 continue; 1282 1283 seq_printf(m, " PML4E #%llu\n", pml4e); 1284 gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m); 1285 } 1286 } else { 1287 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m); 1288 } 1289 } 1290 1291 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt) 1292 { 1293 struct i915_address_space *vm = &ppgtt->base; 1294 struct i915_page_directory_pointer *pdp = &ppgtt->pdp; 1295 struct i915_page_directory *pd; 1296 u64 start = 0, length = ppgtt->base.total; 1297 u64 from = start; 1298 unsigned int pdpe; 1299 1300 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1301 pd = alloc_pd(vm); 1302 if (IS_ERR(pd)) 1303 goto unwind; 1304 1305 gen8_initialize_pd(vm, pd); 1306 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); 1307 pdp->used_pdpes++; 1308 } 1309 1310 pdp->used_pdpes++; /* never remove */ 1311 return 0; 1312 1313 unwind: 1314 start -= from; 1315 gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { 1316 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); 1317 free_pd(vm, pd); 1318 } 1319 pdp->used_pdpes = 0; 1320 return -ENOMEM; 1321 } 1322 1323 /* 1324 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1325 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1326 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1327 * space. 1328 * 1329 */ 1330 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1331 { 1332 struct i915_address_space *vm = &ppgtt->base; 1333 struct drm_i915_private *dev_priv = vm->i915; 1334 int ret; 1335 1336 ppgtt->base.total = USES_FULL_48BIT_PPGTT(dev_priv) ? 1337 1ULL << 48 : 1338 1ULL << 32; 1339 1340 ret = gen8_init_scratch(&ppgtt->base); 1341 if (ret) { 1342 ppgtt->base.total = 0; 1343 return ret; 1344 } 1345 1346 /* There are only few exceptions for gen >=6. chv and bxt. 1347 * And we are not sure about the latter so play safe for now. 1348 */ 1349 if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv)) 1350 ppgtt->base.pt_kmap_wc = true; 1351 1352 if (use_4lvl(vm)) { 1353 ret = setup_px(&ppgtt->base, &ppgtt->pml4); 1354 if (ret) 1355 goto free_scratch; 1356 1357 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); 1358 1359 ppgtt->switch_mm = gen8_mm_switch_4lvl; 1360 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl; 1361 ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl; 1362 ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl; 1363 } else { 1364 ret = __pdp_init(&ppgtt->base, &ppgtt->pdp); 1365 if (ret) 1366 goto free_scratch; 1367 1368 if (intel_vgpu_active(dev_priv)) { 1369 ret = gen8_preallocate_top_level_pdp(ppgtt); 1370 if (ret) { 1371 __pdp_fini(&ppgtt->pdp); 1372 goto free_scratch; 1373 } 1374 } 1375 1376 ppgtt->switch_mm = gen8_mm_switch_3lvl; 1377 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_3lvl; 1378 ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl; 1379 ppgtt->base.clear_range = gen8_ppgtt_clear_3lvl; 1380 } 1381 1382 if (intel_vgpu_active(dev_priv)) 1383 gen8_ppgtt_notify_vgt(ppgtt, true); 1384 1385 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1386 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1387 ppgtt->base.bind_vma = ppgtt_bind_vma; 1388 ppgtt->debug_dump = gen8_dump_ppgtt; 1389 1390 return 0; 1391 1392 free_scratch: 1393 gen8_free_scratch(&ppgtt->base); 1394 return ret; 1395 } 1396 1397 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1398 { 1399 struct i915_address_space *vm = &ppgtt->base; 1400 struct i915_page_table *unused; 1401 gen6_pte_t scratch_pte; 1402 u32 pd_entry, pte, pde; 1403 u32 start = 0, length = ppgtt->base.total; 1404 1405 scratch_pte = vm->pte_encode(vm->scratch_page.daddr, 1406 I915_CACHE_LLC, 0); 1407 1408 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) { 1409 u32 expected; 1410 gen6_pte_t *pt_vaddr; 1411 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1412 pd_entry = readl(ppgtt->pd_addr + pde); 1413 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1414 1415 if (pd_entry != expected) 1416 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1417 pde, 1418 pd_entry, 1419 expected); 1420 seq_printf(m, "\tPDE: %x\n", pd_entry); 1421 1422 pt_vaddr = kmap_atomic_px(ppgtt->pd.page_table[pde]); 1423 1424 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1425 unsigned long va = 1426 (pde * PAGE_SIZE * GEN6_PTES) + 1427 (pte * PAGE_SIZE); 1428 int i; 1429 bool found = false; 1430 for (i = 0; i < 4; i++) 1431 if (pt_vaddr[pte + i] != scratch_pte) 1432 found = true; 1433 if (!found) 1434 continue; 1435 1436 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1437 for (i = 0; i < 4; i++) { 1438 if (pt_vaddr[pte + i] != scratch_pte) 1439 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1440 else 1441 seq_puts(m, " SCRATCH "); 1442 } 1443 seq_puts(m, "\n"); 1444 } 1445 kunmap_atomic(pt_vaddr); 1446 } 1447 } 1448 1449 /* Write pde (index) from the page directory @pd to the page table @pt */ 1450 static inline void gen6_write_pde(const struct i915_hw_ppgtt *ppgtt, 1451 const unsigned int pde, 1452 const struct i915_page_table *pt) 1453 { 1454 /* Caller needs to make sure the write completes if necessary */ 1455 writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID, 1456 ppgtt->pd_addr + pde); 1457 } 1458 1459 /* Write all the page tables found in the ppgtt structure to incrementing page 1460 * directories. */ 1461 static void gen6_write_page_range(struct i915_hw_ppgtt *ppgtt, 1462 u32 start, u32 length) 1463 { 1464 struct i915_page_table *pt; 1465 unsigned int pde; 1466 1467 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) 1468 gen6_write_pde(ppgtt, pde, pt); 1469 1470 mark_tlbs_dirty(ppgtt); 1471 wmb(); 1472 } 1473 1474 static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1475 { 1476 GEM_BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1477 return ppgtt->pd.base.ggtt_offset << 10; 1478 } 1479 1480 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1481 struct drm_i915_gem_request *req) 1482 { 1483 struct intel_engine_cs *engine = req->engine; 1484 u32 *cs; 1485 1486 /* NB: TLBs must be flushed and invalidated before a switch */ 1487 cs = intel_ring_begin(req, 6); 1488 if (IS_ERR(cs)) 1489 return PTR_ERR(cs); 1490 1491 *cs++ = MI_LOAD_REGISTER_IMM(2); 1492 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine)); 1493 *cs++ = PP_DIR_DCLV_2G; 1494 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); 1495 *cs++ = get_pd_offset(ppgtt); 1496 *cs++ = MI_NOOP; 1497 intel_ring_advance(req, cs); 1498 1499 return 0; 1500 } 1501 1502 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1503 struct drm_i915_gem_request *req) 1504 { 1505 struct intel_engine_cs *engine = req->engine; 1506 u32 *cs; 1507 1508 /* NB: TLBs must be flushed and invalidated before a switch */ 1509 cs = intel_ring_begin(req, 6); 1510 if (IS_ERR(cs)) 1511 return PTR_ERR(cs); 1512 1513 *cs++ = MI_LOAD_REGISTER_IMM(2); 1514 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine)); 1515 *cs++ = PP_DIR_DCLV_2G; 1516 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); 1517 *cs++ = get_pd_offset(ppgtt); 1518 *cs++ = MI_NOOP; 1519 intel_ring_advance(req, cs); 1520 1521 return 0; 1522 } 1523 1524 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1525 struct drm_i915_gem_request *req) 1526 { 1527 struct intel_engine_cs *engine = req->engine; 1528 struct drm_i915_private *dev_priv = req->i915; 1529 1530 I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); 1531 I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); 1532 return 0; 1533 } 1534 1535 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv) 1536 { 1537 struct intel_engine_cs *engine; 1538 enum intel_engine_id id; 1539 1540 for_each_engine(engine, dev_priv, id) { 1541 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ? 1542 GEN8_GFX_PPGTT_48B : 0; 1543 I915_WRITE(RING_MODE_GEN7(engine), 1544 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1545 } 1546 } 1547 1548 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv) 1549 { 1550 struct intel_engine_cs *engine; 1551 u32 ecochk, ecobits; 1552 enum intel_engine_id id; 1553 1554 ecobits = I915_READ(GAC_ECO_BITS); 1555 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1556 1557 ecochk = I915_READ(GAM_ECOCHK); 1558 if (IS_HASWELL(dev_priv)) { 1559 ecochk |= ECOCHK_PPGTT_WB_HSW; 1560 } else { 1561 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1562 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1563 } 1564 I915_WRITE(GAM_ECOCHK, ecochk); 1565 1566 for_each_engine(engine, dev_priv, id) { 1567 /* GFX_MODE is per-ring on gen7+ */ 1568 I915_WRITE(RING_MODE_GEN7(engine), 1569 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1570 } 1571 } 1572 1573 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) 1574 { 1575 u32 ecochk, gab_ctl, ecobits; 1576 1577 ecobits = I915_READ(GAC_ECO_BITS); 1578 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1579 ECOBITS_PPGTT_CACHE64B); 1580 1581 gab_ctl = I915_READ(GAB_CTL); 1582 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1583 1584 ecochk = I915_READ(GAM_ECOCHK); 1585 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1586 1587 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1588 } 1589 1590 /* PPGTT support for Sandybdrige/Gen6 and later */ 1591 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1592 u64 start, u64 length) 1593 { 1594 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1595 unsigned int first_entry = start >> PAGE_SHIFT; 1596 unsigned int pde = first_entry / GEN6_PTES; 1597 unsigned int pte = first_entry % GEN6_PTES; 1598 unsigned int num_entries = length >> PAGE_SHIFT; 1599 gen6_pte_t scratch_pte = 1600 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0); 1601 1602 while (num_entries) { 1603 struct i915_page_table *pt = ppgtt->pd.page_table[pde++]; 1604 unsigned int end = min(pte + num_entries, GEN6_PTES); 1605 gen6_pte_t *vaddr; 1606 1607 num_entries -= end - pte; 1608 1609 /* Note that the hw doesn't support removing PDE on the fly 1610 * (they are cached inside the context with no means to 1611 * invalidate the cache), so we can only reset the PTE 1612 * entries back to scratch. 1613 */ 1614 1615 vaddr = kmap_atomic_px(pt); 1616 do { 1617 vaddr[pte++] = scratch_pte; 1618 } while (pte < end); 1619 kunmap_atomic(vaddr); 1620 1621 pte = 0; 1622 } 1623 } 1624 1625 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 1626 struct sg_table *pages, 1627 u64 start, 1628 enum i915_cache_level cache_level, 1629 u32 flags) 1630 { 1631 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1632 unsigned first_entry = start >> PAGE_SHIFT; 1633 unsigned act_pt = first_entry / GEN6_PTES; 1634 unsigned act_pte = first_entry % GEN6_PTES; 1635 const u32 pte_encode = vm->pte_encode(0, cache_level, flags); 1636 struct sgt_dma iter; 1637 gen6_pte_t *vaddr; 1638 1639 vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]); 1640 iter.sg = pages->sgl; 1641 iter.dma = sg_dma_address(iter.sg); 1642 iter.max = iter.dma + iter.sg->length; 1643 do { 1644 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); 1645 1646 iter.dma += PAGE_SIZE; 1647 if (iter.dma == iter.max) { 1648 iter.sg = __sg_next(iter.sg); 1649 if (!iter.sg) 1650 break; 1651 1652 iter.dma = sg_dma_address(iter.sg); 1653 iter.max = iter.dma + iter.sg->length; 1654 } 1655 1656 if (++act_pte == GEN6_PTES) { 1657 kunmap_atomic(vaddr); 1658 vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]); 1659 act_pte = 0; 1660 } 1661 } while (1); 1662 kunmap_atomic(vaddr); 1663 } 1664 1665 static int gen6_alloc_va_range(struct i915_address_space *vm, 1666 u64 start, u64 length) 1667 { 1668 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1669 struct i915_page_table *pt; 1670 u64 from = start; 1671 unsigned int pde; 1672 bool flush = false; 1673 1674 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) { 1675 if (pt == vm->scratch_pt) { 1676 pt = alloc_pt(vm); 1677 if (IS_ERR(pt)) 1678 goto unwind_out; 1679 1680 gen6_initialize_pt(vm, pt); 1681 ppgtt->pd.page_table[pde] = pt; 1682 gen6_write_pde(ppgtt, pde, pt); 1683 flush = true; 1684 } 1685 } 1686 1687 if (flush) { 1688 mark_tlbs_dirty(ppgtt); 1689 wmb(); 1690 } 1691 1692 return 0; 1693 1694 unwind_out: 1695 gen6_ppgtt_clear_range(vm, from, start); 1696 return -ENOMEM; 1697 } 1698 1699 static int gen6_init_scratch(struct i915_address_space *vm) 1700 { 1701 int ret; 1702 1703 ret = setup_scratch_page(vm, I915_GFP_DMA); 1704 if (ret) 1705 return ret; 1706 1707 vm->scratch_pt = alloc_pt(vm); 1708 if (IS_ERR(vm->scratch_pt)) { 1709 cleanup_scratch_page(vm); 1710 return PTR_ERR(vm->scratch_pt); 1711 } 1712 1713 gen6_initialize_pt(vm, vm->scratch_pt); 1714 1715 return 0; 1716 } 1717 1718 static void gen6_free_scratch(struct i915_address_space *vm) 1719 { 1720 free_pt(vm, vm->scratch_pt); 1721 cleanup_scratch_page(vm); 1722 } 1723 1724 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 1725 { 1726 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1727 struct i915_page_directory *pd = &ppgtt->pd; 1728 struct i915_page_table *pt; 1729 u32 pde; 1730 1731 drm_mm_remove_node(&ppgtt->node); 1732 1733 gen6_for_all_pdes(pt, pd, pde) 1734 if (pt != vm->scratch_pt) 1735 free_pt(vm, pt); 1736 1737 gen6_free_scratch(vm); 1738 } 1739 1740 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 1741 { 1742 struct i915_address_space *vm = &ppgtt->base; 1743 struct drm_i915_private *dev_priv = ppgtt->base.i915; 1744 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1745 int ret; 1746 1747 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 1748 * allocator works in address space sizes, so it's multiplied by page 1749 * size. We allocate at the top of the GTT to avoid fragmentation. 1750 */ 1751 BUG_ON(!drm_mm_initialized(&ggtt->base.mm)); 1752 1753 ret = gen6_init_scratch(vm); 1754 if (ret) 1755 return ret; 1756 1757 ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node, 1758 GEN6_PD_SIZE, GEN6_PD_ALIGN, 1759 I915_COLOR_UNEVICTABLE, 1760 0, ggtt->base.total, 1761 PIN_HIGH); 1762 if (ret) 1763 goto err_out; 1764 1765 if (ppgtt->node.start < ggtt->mappable_end) 1766 DRM_DEBUG("Forced to use aperture for PDEs\n"); 1767 1768 ppgtt->pd.base.ggtt_offset = 1769 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 1770 1771 ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + 1772 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 1773 1774 return 0; 1775 1776 err_out: 1777 gen6_free_scratch(vm); 1778 return ret; 1779 } 1780 1781 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 1782 { 1783 return gen6_ppgtt_allocate_page_directories(ppgtt); 1784 } 1785 1786 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 1787 u64 start, u64 length) 1788 { 1789 struct i915_page_table *unused; 1790 u32 pde; 1791 1792 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) 1793 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 1794 } 1795 1796 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1797 { 1798 struct drm_i915_private *dev_priv = ppgtt->base.i915; 1799 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1800 int ret; 1801 1802 ppgtt->base.pte_encode = ggtt->base.pte_encode; 1803 if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv)) 1804 ppgtt->switch_mm = gen6_mm_switch; 1805 else if (IS_HASWELL(dev_priv)) 1806 ppgtt->switch_mm = hsw_mm_switch; 1807 else if (IS_GEN7(dev_priv)) 1808 ppgtt->switch_mm = gen7_mm_switch; 1809 else 1810 BUG(); 1811 1812 ret = gen6_ppgtt_alloc(ppgtt); 1813 if (ret) 1814 return ret; 1815 1816 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 1817 1818 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 1819 gen6_write_page_range(ppgtt, 0, ppgtt->base.total); 1820 1821 ret = gen6_alloc_va_range(&ppgtt->base, 0, ppgtt->base.total); 1822 if (ret) { 1823 gen6_ppgtt_cleanup(&ppgtt->base); 1824 return ret; 1825 } 1826 1827 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 1828 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 1829 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1830 ppgtt->base.bind_vma = ppgtt_bind_vma; 1831 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 1832 ppgtt->debug_dump = gen6_dump_ppgtt; 1833 1834 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", 1835 ppgtt->node.size >> 20, 1836 ppgtt->node.start / PAGE_SIZE); 1837 1838 DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n", 1839 ppgtt->pd.base.ggtt_offset << 10); 1840 1841 return 0; 1842 } 1843 1844 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt, 1845 struct drm_i915_private *dev_priv) 1846 { 1847 ppgtt->base.i915 = dev_priv; 1848 ppgtt->base.dma = &dev_priv->drm.pdev->dev; 1849 1850 if (INTEL_INFO(dev_priv)->gen < 8) 1851 return gen6_ppgtt_init(ppgtt); 1852 else 1853 return gen8_ppgtt_init(ppgtt); 1854 } 1855 1856 static void i915_address_space_init(struct i915_address_space *vm, 1857 struct drm_i915_private *dev_priv, 1858 const char *name) 1859 { 1860 i915_gem_timeline_init(dev_priv, &vm->timeline, name); 1861 1862 drm_mm_init(&vm->mm, 0, vm->total); 1863 vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; 1864 1865 INIT_LIST_HEAD(&vm->active_list); 1866 INIT_LIST_HEAD(&vm->inactive_list); 1867 INIT_LIST_HEAD(&vm->unbound_list); 1868 1869 list_add_tail(&vm->global_link, &dev_priv->vm_list); 1870 pagevec_init(&vm->free_pages, false); 1871 } 1872 1873 static void i915_address_space_fini(struct i915_address_space *vm) 1874 { 1875 if (pagevec_count(&vm->free_pages)) 1876 vm_free_pages_release(vm); 1877 1878 i915_gem_timeline_fini(&vm->timeline); 1879 drm_mm_takedown(&vm->mm); 1880 list_del(&vm->global_link); 1881 } 1882 1883 static void gtt_write_workarounds(struct drm_i915_private *dev_priv) 1884 { 1885 /* This function is for gtt related workarounds. This function is 1886 * called on driver load and after a GPU reset, so you can place 1887 * workarounds here even if they get overwritten by GPU reset. 1888 */ 1889 /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk */ 1890 if (IS_BROADWELL(dev_priv)) 1891 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW); 1892 else if (IS_CHERRYVIEW(dev_priv)) 1893 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV); 1894 else if (IS_GEN9_BC(dev_priv)) 1895 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL); 1896 else if (IS_GEN9_LP(dev_priv)) 1897 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT); 1898 } 1899 1900 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv) 1901 { 1902 gtt_write_workarounds(dev_priv); 1903 1904 /* In the case of execlists, PPGTT is enabled by the context descriptor 1905 * and the PDPs are contained within the context itself. We don't 1906 * need to do anything here. */ 1907 if (i915.enable_execlists) 1908 return 0; 1909 1910 if (!USES_PPGTT(dev_priv)) 1911 return 0; 1912 1913 if (IS_GEN6(dev_priv)) 1914 gen6_ppgtt_enable(dev_priv); 1915 else if (IS_GEN7(dev_priv)) 1916 gen7_ppgtt_enable(dev_priv); 1917 else if (INTEL_GEN(dev_priv) >= 8) 1918 gen8_ppgtt_enable(dev_priv); 1919 else 1920 MISSING_CASE(INTEL_GEN(dev_priv)); 1921 1922 return 0; 1923 } 1924 1925 struct i915_hw_ppgtt * 1926 i915_ppgtt_create(struct drm_i915_private *dev_priv, 1927 struct drm_i915_file_private *fpriv, 1928 const char *name) 1929 { 1930 struct i915_hw_ppgtt *ppgtt; 1931 int ret; 1932 1933 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 1934 if (!ppgtt) 1935 return ERR_PTR(-ENOMEM); 1936 1937 ret = __hw_ppgtt_init(ppgtt, dev_priv); 1938 if (ret) { 1939 kfree(ppgtt); 1940 return ERR_PTR(ret); 1941 } 1942 1943 kref_init(&ppgtt->ref); 1944 i915_address_space_init(&ppgtt->base, dev_priv, name); 1945 ppgtt->base.file = fpriv; 1946 1947 trace_i915_ppgtt_create(&ppgtt->base); 1948 1949 return ppgtt; 1950 } 1951 1952 void i915_ppgtt_close(struct i915_address_space *vm) 1953 { 1954 struct list_head *phases[] = { 1955 &vm->active_list, 1956 &vm->inactive_list, 1957 &vm->unbound_list, 1958 NULL, 1959 }, **phase; 1960 1961 GEM_BUG_ON(vm->closed); 1962 vm->closed = true; 1963 1964 for (phase = phases; *phase; phase++) { 1965 struct i915_vma *vma, *vn; 1966 1967 list_for_each_entry_safe(vma, vn, *phase, vm_link) 1968 if (!i915_vma_is_closed(vma)) 1969 i915_vma_close(vma); 1970 } 1971 } 1972 1973 void i915_ppgtt_release(struct kref *kref) 1974 { 1975 struct i915_hw_ppgtt *ppgtt = 1976 container_of(kref, struct i915_hw_ppgtt, ref); 1977 1978 trace_i915_ppgtt_release(&ppgtt->base); 1979 1980 /* vmas should already be unbound and destroyed */ 1981 WARN_ON(!list_empty(&ppgtt->base.active_list)); 1982 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 1983 WARN_ON(!list_empty(&ppgtt->base.unbound_list)); 1984 1985 ppgtt->base.cleanup(&ppgtt->base); 1986 i915_address_space_fini(&ppgtt->base); 1987 kfree(ppgtt); 1988 } 1989 1990 /* Certain Gen5 chipsets require require idling the GPU before 1991 * unmapping anything from the GTT when VT-d is enabled. 1992 */ 1993 static bool needs_idle_maps(struct drm_i915_private *dev_priv) 1994 { 1995 #ifdef CONFIG_INTEL_IOMMU 1996 /* Query intel_iommu to see if we need the workaround. Presumably that 1997 * was loaded first. 1998 */ 1999 if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped) 2000 return true; 2001 #endif 2002 return false; 2003 } 2004 2005 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv) 2006 { 2007 struct intel_engine_cs *engine; 2008 enum intel_engine_id id; 2009 2010 if (INTEL_INFO(dev_priv)->gen < 6) 2011 return; 2012 2013 for_each_engine(engine, dev_priv, id) { 2014 u32 fault_reg; 2015 fault_reg = I915_READ(RING_FAULT_REG(engine)); 2016 if (fault_reg & RING_FAULT_VALID) { 2017 DRM_DEBUG_DRIVER("Unexpected fault\n" 2018 "\tAddr: 0x%08lx\n" 2019 "\tAddress space: %s\n" 2020 "\tSource ID: %d\n" 2021 "\tType: %d\n", 2022 fault_reg & PAGE_MASK, 2023 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 2024 RING_FAULT_SRCID(fault_reg), 2025 RING_FAULT_FAULT_TYPE(fault_reg)); 2026 I915_WRITE(RING_FAULT_REG(engine), 2027 fault_reg & ~RING_FAULT_VALID); 2028 } 2029 } 2030 2031 /* Engine specific init may not have been done till this point. */ 2032 if (dev_priv->engine[RCS]) 2033 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS])); 2034 } 2035 2036 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv) 2037 { 2038 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2039 2040 /* Don't bother messing with faults pre GEN6 as we have little 2041 * documentation supporting that it's a good idea. 2042 */ 2043 if (INTEL_GEN(dev_priv) < 6) 2044 return; 2045 2046 i915_check_and_clear_faults(dev_priv); 2047 2048 ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total); 2049 2050 i915_ggtt_invalidate(dev_priv); 2051 } 2052 2053 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, 2054 struct sg_table *pages) 2055 { 2056 do { 2057 if (dma_map_sg(&obj->base.dev->pdev->dev, 2058 pages->sgl, pages->nents, 2059 PCI_DMA_BIDIRECTIONAL)) 2060 return 0; 2061 2062 /* If the DMA remap fails, one cause can be that we have 2063 * too many objects pinned in a small remapping table, 2064 * such as swiotlb. Incrementally purge all other objects and 2065 * try again - if there are no more pages to remove from 2066 * the DMA remapper, i915_gem_shrink will return 0. 2067 */ 2068 GEM_BUG_ON(obj->mm.pages == pages); 2069 } while (i915_gem_shrink(to_i915(obj->base.dev), 2070 obj->base.size >> PAGE_SHIFT, 2071 I915_SHRINK_BOUND | 2072 I915_SHRINK_UNBOUND | 2073 I915_SHRINK_ACTIVE)); 2074 2075 return -ENOSPC; 2076 } 2077 2078 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2079 { 2080 writeq(pte, addr); 2081 } 2082 2083 static void gen8_ggtt_insert_page(struct i915_address_space *vm, 2084 dma_addr_t addr, 2085 u64 offset, 2086 enum i915_cache_level level, 2087 u32 unused) 2088 { 2089 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2090 gen8_pte_t __iomem *pte = 2091 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT); 2092 2093 gen8_set_pte(pte, gen8_pte_encode(addr, level)); 2094 2095 ggtt->invalidate(vm->i915); 2096 } 2097 2098 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2099 struct sg_table *st, 2100 u64 start, 2101 enum i915_cache_level level, 2102 u32 unused) 2103 { 2104 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2105 struct sgt_iter sgt_iter; 2106 gen8_pte_t __iomem *gtt_entries; 2107 const gen8_pte_t pte_encode = gen8_pte_encode(0, level); 2108 dma_addr_t addr; 2109 2110 gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm; 2111 gtt_entries += start >> PAGE_SHIFT; 2112 for_each_sgt_dma(addr, sgt_iter, st) 2113 gen8_set_pte(gtt_entries++, pte_encode | addr); 2114 2115 wmb(); 2116 2117 /* This next bit makes the above posting read even more important. We 2118 * want to flush the TLBs only after we're certain all the PTE updates 2119 * have finished. 2120 */ 2121 ggtt->invalidate(vm->i915); 2122 } 2123 2124 static void gen6_ggtt_insert_page(struct i915_address_space *vm, 2125 dma_addr_t addr, 2126 u64 offset, 2127 enum i915_cache_level level, 2128 u32 flags) 2129 { 2130 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2131 gen6_pte_t __iomem *pte = 2132 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT); 2133 2134 iowrite32(vm->pte_encode(addr, level, flags), pte); 2135 2136 ggtt->invalidate(vm->i915); 2137 } 2138 2139 /* 2140 * Binds an object into the global gtt with the specified cache level. The object 2141 * will be accessible to the GPU via commands whose operands reference offsets 2142 * within the global GTT as well as accessible by the GPU through the GMADR 2143 * mapped BAR (dev_priv->mm.gtt->gtt). 2144 */ 2145 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 2146 struct sg_table *st, 2147 u64 start, 2148 enum i915_cache_level level, 2149 u32 flags) 2150 { 2151 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2152 gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm; 2153 unsigned int i = start >> PAGE_SHIFT; 2154 struct sgt_iter iter; 2155 dma_addr_t addr; 2156 for_each_sgt_dma(addr, iter, st) 2157 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]); 2158 wmb(); 2159 2160 /* This next bit makes the above posting read even more important. We 2161 * want to flush the TLBs only after we're certain all the PTE updates 2162 * have finished. 2163 */ 2164 ggtt->invalidate(vm->i915); 2165 } 2166 2167 static void nop_clear_range(struct i915_address_space *vm, 2168 u64 start, u64 length) 2169 { 2170 } 2171 2172 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2173 u64 start, u64 length) 2174 { 2175 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2176 unsigned first_entry = start >> PAGE_SHIFT; 2177 unsigned num_entries = length >> PAGE_SHIFT; 2178 const gen8_pte_t scratch_pte = 2179 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); 2180 gen8_pte_t __iomem *gtt_base = 2181 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 2182 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 2183 int i; 2184 2185 if (WARN(num_entries > max_entries, 2186 "First entry = %d; Num entries = %d (max=%d)\n", 2187 first_entry, num_entries, max_entries)) 2188 num_entries = max_entries; 2189 2190 for (i = 0; i < num_entries; i++) 2191 gen8_set_pte(>t_base[i], scratch_pte); 2192 } 2193 2194 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 2195 { 2196 struct drm_i915_private *dev_priv = vm->i915; 2197 2198 /* 2199 * Make sure the internal GAM fifo has been cleared of all GTT 2200 * writes before exiting stop_machine(). This guarantees that 2201 * any aperture accesses waiting to start in another process 2202 * cannot back up behind the GTT writes causing a hang. 2203 * The register can be any arbitrary GAM register. 2204 */ 2205 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2206 } 2207 2208 struct insert_page { 2209 struct i915_address_space *vm; 2210 dma_addr_t addr; 2211 u64 offset; 2212 enum i915_cache_level level; 2213 }; 2214 2215 static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 2216 { 2217 struct insert_page *arg = _arg; 2218 2219 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 2220 bxt_vtd_ggtt_wa(arg->vm); 2221 2222 return 0; 2223 } 2224 2225 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 2226 dma_addr_t addr, 2227 u64 offset, 2228 enum i915_cache_level level, 2229 u32 unused) 2230 { 2231 struct insert_page arg = { vm, addr, offset, level }; 2232 2233 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 2234 } 2235 2236 struct insert_entries { 2237 struct i915_address_space *vm; 2238 struct sg_table *st; 2239 u64 start; 2240 enum i915_cache_level level; 2241 }; 2242 2243 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 2244 { 2245 struct insert_entries *arg = _arg; 2246 2247 gen8_ggtt_insert_entries(arg->vm, arg->st, arg->start, arg->level, 0); 2248 bxt_vtd_ggtt_wa(arg->vm); 2249 2250 return 0; 2251 } 2252 2253 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 2254 struct sg_table *st, 2255 u64 start, 2256 enum i915_cache_level level, 2257 u32 unused) 2258 { 2259 struct insert_entries arg = { vm, st, start, level }; 2260 2261 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 2262 } 2263 2264 struct clear_range { 2265 struct i915_address_space *vm; 2266 u64 start; 2267 u64 length; 2268 }; 2269 2270 static int bxt_vtd_ggtt_clear_range__cb(void *_arg) 2271 { 2272 struct clear_range *arg = _arg; 2273 2274 gen8_ggtt_clear_range(arg->vm, arg->start, arg->length); 2275 bxt_vtd_ggtt_wa(arg->vm); 2276 2277 return 0; 2278 } 2279 2280 static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm, 2281 u64 start, 2282 u64 length) 2283 { 2284 struct clear_range arg = { vm, start, length }; 2285 2286 stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL); 2287 } 2288 2289 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2290 u64 start, u64 length) 2291 { 2292 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2293 unsigned first_entry = start >> PAGE_SHIFT; 2294 unsigned num_entries = length >> PAGE_SHIFT; 2295 gen6_pte_t scratch_pte, __iomem *gtt_base = 2296 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 2297 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 2298 int i; 2299 2300 if (WARN(num_entries > max_entries, 2301 "First entry = %d; Num entries = %d (max=%d)\n", 2302 first_entry, num_entries, max_entries)) 2303 num_entries = max_entries; 2304 2305 scratch_pte = vm->pte_encode(vm->scratch_page.daddr, 2306 I915_CACHE_LLC, 0); 2307 2308 for (i = 0; i < num_entries; i++) 2309 iowrite32(scratch_pte, >t_base[i]); 2310 } 2311 2312 static void i915_ggtt_insert_page(struct i915_address_space *vm, 2313 dma_addr_t addr, 2314 u64 offset, 2315 enum i915_cache_level cache_level, 2316 u32 unused) 2317 { 2318 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2319 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2320 2321 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); 2322 } 2323 2324 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2325 struct sg_table *pages, 2326 u64 start, 2327 enum i915_cache_level cache_level, 2328 u32 unused) 2329 { 2330 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2331 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2332 2333 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2334 } 2335 2336 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2337 u64 start, u64 length) 2338 { 2339 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); 2340 } 2341 2342 static int ggtt_bind_vma(struct i915_vma *vma, 2343 enum i915_cache_level cache_level, 2344 u32 flags) 2345 { 2346 struct drm_i915_private *i915 = vma->vm->i915; 2347 struct drm_i915_gem_object *obj = vma->obj; 2348 u32 pte_flags; 2349 2350 if (unlikely(!vma->pages)) { 2351 int ret = i915_get_ggtt_vma_pages(vma); 2352 if (ret) 2353 return ret; 2354 } 2355 2356 /* Currently applicable only to VLV */ 2357 pte_flags = 0; 2358 if (obj->gt_ro) 2359 pte_flags |= PTE_READ_ONLY; 2360 2361 intel_runtime_pm_get(i915); 2362 vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, 2363 cache_level, pte_flags); 2364 intel_runtime_pm_put(i915); 2365 2366 /* 2367 * Without aliasing PPGTT there's no difference between 2368 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 2369 * upgrade to both bound if we bind either to avoid double-binding. 2370 */ 2371 vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 2372 2373 return 0; 2374 } 2375 2376 static void ggtt_unbind_vma(struct i915_vma *vma) 2377 { 2378 struct drm_i915_private *i915 = vma->vm->i915; 2379 2380 intel_runtime_pm_get(i915); 2381 vma->vm->clear_range(vma->vm, vma->node.start, vma->size); 2382 intel_runtime_pm_put(i915); 2383 } 2384 2385 static int aliasing_gtt_bind_vma(struct i915_vma *vma, 2386 enum i915_cache_level cache_level, 2387 u32 flags) 2388 { 2389 struct drm_i915_private *i915 = vma->vm->i915; 2390 u32 pte_flags; 2391 int ret; 2392 2393 if (unlikely(!vma->pages)) { 2394 ret = i915_get_ggtt_vma_pages(vma); 2395 if (ret) 2396 return ret; 2397 } 2398 2399 /* Currently applicable only to VLV */ 2400 pte_flags = 0; 2401 if (vma->obj->gt_ro) 2402 pte_flags |= PTE_READ_ONLY; 2403 2404 if (flags & I915_VMA_LOCAL_BIND) { 2405 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; 2406 2407 if (!(vma->flags & I915_VMA_LOCAL_BIND) && 2408 appgtt->base.allocate_va_range) { 2409 ret = appgtt->base.allocate_va_range(&appgtt->base, 2410 vma->node.start, 2411 vma->size); 2412 if (ret) 2413 goto err_pages; 2414 } 2415 2416 appgtt->base.insert_entries(&appgtt->base, 2417 vma->pages, vma->node.start, 2418 cache_level, pte_flags); 2419 } 2420 2421 if (flags & I915_VMA_GLOBAL_BIND) { 2422 intel_runtime_pm_get(i915); 2423 vma->vm->insert_entries(vma->vm, 2424 vma->pages, vma->node.start, 2425 cache_level, pte_flags); 2426 intel_runtime_pm_put(i915); 2427 } 2428 2429 return 0; 2430 2431 err_pages: 2432 if (!(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))) { 2433 if (vma->pages != vma->obj->mm.pages) { 2434 GEM_BUG_ON(!vma->pages); 2435 sg_free_table(vma->pages); 2436 kfree(vma->pages); 2437 } 2438 vma->pages = NULL; 2439 } 2440 return ret; 2441 } 2442 2443 static void aliasing_gtt_unbind_vma(struct i915_vma *vma) 2444 { 2445 struct drm_i915_private *i915 = vma->vm->i915; 2446 2447 if (vma->flags & I915_VMA_GLOBAL_BIND) { 2448 intel_runtime_pm_get(i915); 2449 vma->vm->clear_range(vma->vm, vma->node.start, vma->size); 2450 intel_runtime_pm_put(i915); 2451 } 2452 2453 if (vma->flags & I915_VMA_LOCAL_BIND) { 2454 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base; 2455 2456 vm->clear_range(vm, vma->node.start, vma->size); 2457 } 2458 } 2459 2460 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, 2461 struct sg_table *pages) 2462 { 2463 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 2464 struct device *kdev = &dev_priv->drm.pdev->dev; 2465 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2466 2467 if (unlikely(ggtt->do_idle_maps)) { 2468 if (i915_gem_wait_for_idle(dev_priv, 0)) { 2469 DRM_ERROR("Failed to wait for idle; VT'd may hang.\n"); 2470 /* Wait a bit, in hopes it avoids the hang */ 2471 udelay(10); 2472 } 2473 } 2474 2475 dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL); 2476 } 2477 2478 static void i915_gtt_color_adjust(const struct drm_mm_node *node, 2479 unsigned long color, 2480 u64 *start, 2481 u64 *end) 2482 { 2483 if (node->allocated && node->color != color) 2484 *start += I915_GTT_PAGE_SIZE; 2485 2486 /* Also leave a space between the unallocated reserved node after the 2487 * GTT and any objects within the GTT, i.e. we use the color adjustment 2488 * to insert a guard page to prevent prefetches crossing over the 2489 * GTT boundary. 2490 */ 2491 node = list_next_entry(node, node_list); 2492 if (node->color != color) 2493 *end -= I915_GTT_PAGE_SIZE; 2494 } 2495 2496 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915) 2497 { 2498 struct i915_ggtt *ggtt = &i915->ggtt; 2499 struct i915_hw_ppgtt *ppgtt; 2500 int err; 2501 2502 ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]"); 2503 if (IS_ERR(ppgtt)) 2504 return PTR_ERR(ppgtt); 2505 2506 if (WARN_ON(ppgtt->base.total < ggtt->base.total)) { 2507 err = -ENODEV; 2508 goto err_ppgtt; 2509 } 2510 2511 if (ppgtt->base.allocate_va_range) { 2512 /* Note we only pre-allocate as far as the end of the global 2513 * GTT. On 48b / 4-level page-tables, the difference is very, 2514 * very significant! We have to preallocate as GVT/vgpu does 2515 * not like the page directory disappearing. 2516 */ 2517 err = ppgtt->base.allocate_va_range(&ppgtt->base, 2518 0, ggtt->base.total); 2519 if (err) 2520 goto err_ppgtt; 2521 } 2522 2523 i915->mm.aliasing_ppgtt = ppgtt; 2524 2525 WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma); 2526 ggtt->base.bind_vma = aliasing_gtt_bind_vma; 2527 2528 WARN_ON(ggtt->base.unbind_vma != ggtt_unbind_vma); 2529 ggtt->base.unbind_vma = aliasing_gtt_unbind_vma; 2530 2531 return 0; 2532 2533 err_ppgtt: 2534 i915_ppgtt_put(ppgtt); 2535 return err; 2536 } 2537 2538 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915) 2539 { 2540 struct i915_ggtt *ggtt = &i915->ggtt; 2541 struct i915_hw_ppgtt *ppgtt; 2542 2543 ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt); 2544 if (!ppgtt) 2545 return; 2546 2547 i915_ppgtt_put(ppgtt); 2548 2549 ggtt->base.bind_vma = ggtt_bind_vma; 2550 ggtt->base.unbind_vma = ggtt_unbind_vma; 2551 } 2552 2553 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) 2554 { 2555 /* Let GEM Manage all of the aperture. 2556 * 2557 * However, leave one page at the end still bound to the scratch page. 2558 * There are a number of places where the hardware apparently prefetches 2559 * past the end of the object, and we've seen multiple hangs with the 2560 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2561 * aperture. One page should be enough to keep any prefetching inside 2562 * of the aperture. 2563 */ 2564 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2565 unsigned long hole_start, hole_end; 2566 struct drm_mm_node *entry; 2567 int ret; 2568 2569 ret = intel_vgt_balloon(dev_priv); 2570 if (ret) 2571 return ret; 2572 2573 /* Reserve a mappable slot for our lockless error capture */ 2574 ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture, 2575 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, 2576 0, ggtt->mappable_end, 2577 DRM_MM_INSERT_LOW); 2578 if (ret) 2579 return ret; 2580 2581 /* Clear any non-preallocated blocks */ 2582 drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) { 2583 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 2584 hole_start, hole_end); 2585 ggtt->base.clear_range(&ggtt->base, hole_start, 2586 hole_end - hole_start); 2587 } 2588 2589 /* And finally clear the reserved guard page */ 2590 ggtt->base.clear_range(&ggtt->base, 2591 ggtt->base.total - PAGE_SIZE, PAGE_SIZE); 2592 2593 if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) { 2594 ret = i915_gem_init_aliasing_ppgtt(dev_priv); 2595 if (ret) 2596 goto err; 2597 } 2598 2599 return 0; 2600 2601 err: 2602 drm_mm_remove_node(&ggtt->error_capture); 2603 return ret; 2604 } 2605 2606 /** 2607 * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization 2608 * @dev_priv: i915 device 2609 */ 2610 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv) 2611 { 2612 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2613 struct i915_vma *vma, *vn; 2614 2615 ggtt->base.closed = true; 2616 2617 mutex_lock(&dev_priv->drm.struct_mutex); 2618 WARN_ON(!list_empty(&ggtt->base.active_list)); 2619 list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link) 2620 WARN_ON(i915_vma_unbind(vma)); 2621 mutex_unlock(&dev_priv->drm.struct_mutex); 2622 2623 i915_gem_cleanup_stolen(&dev_priv->drm); 2624 2625 mutex_lock(&dev_priv->drm.struct_mutex); 2626 i915_gem_fini_aliasing_ppgtt(dev_priv); 2627 2628 if (drm_mm_node_allocated(&ggtt->error_capture)) 2629 drm_mm_remove_node(&ggtt->error_capture); 2630 2631 if (drm_mm_initialized(&ggtt->base.mm)) { 2632 intel_vgt_deballoon(dev_priv); 2633 i915_address_space_fini(&ggtt->base); 2634 } 2635 2636 ggtt->base.cleanup(&ggtt->base); 2637 mutex_unlock(&dev_priv->drm.struct_mutex); 2638 2639 arch_phys_wc_del(ggtt->mtrr); 2640 io_mapping_fini(&ggtt->mappable); 2641 } 2642 2643 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 2644 { 2645 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 2646 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 2647 return snb_gmch_ctl << 20; 2648 } 2649 2650 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 2651 { 2652 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 2653 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 2654 if (bdw_gmch_ctl) 2655 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 2656 2657 #ifdef CONFIG_X86_32 2658 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 2659 if (bdw_gmch_ctl > 4) 2660 bdw_gmch_ctl = 4; 2661 #endif 2662 2663 return bdw_gmch_ctl << 20; 2664 } 2665 2666 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 2667 { 2668 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 2669 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 2670 2671 if (gmch_ctrl) 2672 return 1 << (20 + gmch_ctrl); 2673 2674 return 0; 2675 } 2676 2677 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 2678 { 2679 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 2680 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 2681 return snb_gmch_ctl << 25; /* 32 MB units */ 2682 } 2683 2684 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 2685 { 2686 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2687 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 2688 return bdw_gmch_ctl << 25; /* 32 MB units */ 2689 } 2690 2691 static size_t chv_get_stolen_size(u16 gmch_ctrl) 2692 { 2693 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 2694 gmch_ctrl &= SNB_GMCH_GMS_MASK; 2695 2696 /* 2697 * 0x0 to 0x10: 32MB increments starting at 0MB 2698 * 0x11 to 0x16: 4MB increments starting at 8MB 2699 * 0x17 to 0x1d: 4MB increments start at 36MB 2700 */ 2701 if (gmch_ctrl < 0x11) 2702 return gmch_ctrl << 25; 2703 else if (gmch_ctrl < 0x17) 2704 return (gmch_ctrl - 0x11 + 2) << 22; 2705 else 2706 return (gmch_ctrl - 0x17 + 9) << 22; 2707 } 2708 2709 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 2710 { 2711 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2712 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 2713 2714 if (gen9_gmch_ctl < 0xf0) 2715 return gen9_gmch_ctl << 25; /* 32 MB units */ 2716 else 2717 /* 4MB increments starting at 0xf0 for 4MB */ 2718 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 2719 } 2720 2721 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 2722 { 2723 struct drm_i915_private *dev_priv = ggtt->base.i915; 2724 struct pci_dev *pdev = dev_priv->drm.pdev; 2725 phys_addr_t phys_addr; 2726 int ret; 2727 2728 /* For Modern GENs the PTEs and register space are split in the BAR */ 2729 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; 2730 2731 /* 2732 * On BXT writes larger than 64 bit to the GTT pagetable range will be 2733 * dropped. For WC mappings in general we have 64 byte burst writes 2734 * when the WC buffer is flushed, so we can't use it, but have to 2735 * resort to an uncached mapping. The WC issue is easily caught by the 2736 * readback check when writing GTT PTE entries. 2737 */ 2738 if (IS_GEN9_LP(dev_priv)) 2739 ggtt->gsm = ioremap_nocache(phys_addr, size); 2740 else 2741 ggtt->gsm = ioremap_wc(phys_addr, size); 2742 if (!ggtt->gsm) { 2743 DRM_ERROR("Failed to map the ggtt page table\n"); 2744 return -ENOMEM; 2745 } 2746 2747 ret = setup_scratch_page(&ggtt->base, GFP_DMA32); 2748 if (ret) { 2749 DRM_ERROR("Scratch setup failed\n"); 2750 /* iounmap will also get called at remove, but meh */ 2751 iounmap(ggtt->gsm); 2752 return ret; 2753 } 2754 2755 return 0; 2756 } 2757 2758 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 2759 * bits. When using advanced contexts each context stores its own PAT, but 2760 * writing this data shouldn't be harmful even in those cases. */ 2761 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 2762 { 2763 u64 pat; 2764 2765 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 2766 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 2767 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 2768 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 2769 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 2770 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 2771 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 2772 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 2773 2774 if (!USES_PPGTT(dev_priv)) 2775 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 2776 * so RTL will always use the value corresponding to 2777 * pat_sel = 000". 2778 * So let's disable cache for GGTT to avoid screen corruptions. 2779 * MOCS still can be used though. 2780 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 2781 * before this patch, i.e. the same uncached + snooping access 2782 * like on gen6/7 seems to be in effect. 2783 * - So this just fixes blitter/render access. Again it looks 2784 * like it's not just uncached access, but uncached + snooping. 2785 * So we can still hold onto all our assumptions wrt cpu 2786 * clflushing on LLC machines. 2787 */ 2788 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 2789 2790 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 2791 * write would work. */ 2792 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2793 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2794 } 2795 2796 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 2797 { 2798 u64 pat; 2799 2800 /* 2801 * Map WB on BDW to snooped on CHV. 2802 * 2803 * Only the snoop bit has meaning for CHV, the rest is 2804 * ignored. 2805 * 2806 * The hardware will never snoop for certain types of accesses: 2807 * - CPU GTT (GMADR->GGTT->no snoop->memory) 2808 * - PPGTT page tables 2809 * - some other special cycles 2810 * 2811 * As with BDW, we also need to consider the following for GT accesses: 2812 * "For GGTT, there is NO pat_sel[2:0] from the entry, 2813 * so RTL will always use the value corresponding to 2814 * pat_sel = 000". 2815 * Which means we must set the snoop bit in PAT entry 0 2816 * in order to keep the global status page working. 2817 */ 2818 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 2819 GEN8_PPAT(1, 0) | 2820 GEN8_PPAT(2, 0) | 2821 GEN8_PPAT(3, 0) | 2822 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 2823 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 2824 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 2825 GEN8_PPAT(7, CHV_PPAT_SNOOP); 2826 2827 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2828 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2829 } 2830 2831 static void gen6_gmch_remove(struct i915_address_space *vm) 2832 { 2833 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2834 2835 iounmap(ggtt->gsm); 2836 cleanup_scratch_page(vm); 2837 } 2838 2839 static int gen8_gmch_probe(struct i915_ggtt *ggtt) 2840 { 2841 struct drm_i915_private *dev_priv = ggtt->base.i915; 2842 struct pci_dev *pdev = dev_priv->drm.pdev; 2843 unsigned int size; 2844 u16 snb_gmch_ctl; 2845 2846 /* TODO: We're not aware of mappable constraints on gen8 yet */ 2847 ggtt->mappable_base = pci_resource_start(pdev, 2); 2848 ggtt->mappable_end = pci_resource_len(pdev, 2); 2849 2850 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39))) 2851 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39)); 2852 2853 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2854 2855 if (INTEL_GEN(dev_priv) >= 9) { 2856 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl); 2857 size = gen8_get_total_gtt_size(snb_gmch_ctl); 2858 } else if (IS_CHERRYVIEW(dev_priv)) { 2859 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl); 2860 size = chv_get_total_gtt_size(snb_gmch_ctl); 2861 } else { 2862 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl); 2863 size = gen8_get_total_gtt_size(snb_gmch_ctl); 2864 } 2865 2866 ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 2867 2868 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv)) 2869 chv_setup_private_ppat(dev_priv); 2870 else 2871 bdw_setup_private_ppat(dev_priv); 2872 2873 ggtt->base.cleanup = gen6_gmch_remove; 2874 ggtt->base.bind_vma = ggtt_bind_vma; 2875 ggtt->base.unbind_vma = ggtt_unbind_vma; 2876 ggtt->base.insert_page = gen8_ggtt_insert_page; 2877 ggtt->base.clear_range = nop_clear_range; 2878 if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv)) 2879 ggtt->base.clear_range = gen8_ggtt_clear_range; 2880 2881 ggtt->base.insert_entries = gen8_ggtt_insert_entries; 2882 2883 /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ 2884 if (intel_ggtt_update_needs_vtd_wa(dev_priv)) { 2885 ggtt->base.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 2886 ggtt->base.insert_page = bxt_vtd_ggtt_insert_page__BKL; 2887 if (ggtt->base.clear_range != nop_clear_range) 2888 ggtt->base.clear_range = bxt_vtd_ggtt_clear_range__BKL; 2889 } 2890 2891 ggtt->invalidate = gen6_ggtt_invalidate; 2892 2893 return ggtt_probe_common(ggtt, size); 2894 } 2895 2896 static int gen6_gmch_probe(struct i915_ggtt *ggtt) 2897 { 2898 struct drm_i915_private *dev_priv = ggtt->base.i915; 2899 struct pci_dev *pdev = dev_priv->drm.pdev; 2900 unsigned int size; 2901 u16 snb_gmch_ctl; 2902 2903 ggtt->mappable_base = pci_resource_start(pdev, 2); 2904 ggtt->mappable_end = pci_resource_len(pdev, 2); 2905 2906 /* 64/512MB is the current min/max we actually know of, but this is just 2907 * a coarse sanity check. 2908 */ 2909 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { 2910 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end); 2911 return -ENXIO; 2912 } 2913 2914 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40))) 2915 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40)); 2916 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2917 2918 ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); 2919 2920 size = gen6_get_total_gtt_size(snb_gmch_ctl); 2921 ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 2922 2923 ggtt->base.clear_range = gen6_ggtt_clear_range; 2924 ggtt->base.insert_page = gen6_ggtt_insert_page; 2925 ggtt->base.insert_entries = gen6_ggtt_insert_entries; 2926 ggtt->base.bind_vma = ggtt_bind_vma; 2927 ggtt->base.unbind_vma = ggtt_unbind_vma; 2928 ggtt->base.cleanup = gen6_gmch_remove; 2929 2930 ggtt->invalidate = gen6_ggtt_invalidate; 2931 2932 if (HAS_EDRAM(dev_priv)) 2933 ggtt->base.pte_encode = iris_pte_encode; 2934 else if (IS_HASWELL(dev_priv)) 2935 ggtt->base.pte_encode = hsw_pte_encode; 2936 else if (IS_VALLEYVIEW(dev_priv)) 2937 ggtt->base.pte_encode = byt_pte_encode; 2938 else if (INTEL_GEN(dev_priv) >= 7) 2939 ggtt->base.pte_encode = ivb_pte_encode; 2940 else 2941 ggtt->base.pte_encode = snb_pte_encode; 2942 2943 return ggtt_probe_common(ggtt, size); 2944 } 2945 2946 static void i915_gmch_remove(struct i915_address_space *vm) 2947 { 2948 intel_gmch_remove(); 2949 } 2950 2951 static int i915_gmch_probe(struct i915_ggtt *ggtt) 2952 { 2953 struct drm_i915_private *dev_priv = ggtt->base.i915; 2954 int ret; 2955 2956 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL); 2957 if (!ret) { 2958 DRM_ERROR("failed to set up gmch\n"); 2959 return -EIO; 2960 } 2961 2962 intel_gtt_get(&ggtt->base.total, 2963 &ggtt->stolen_size, 2964 &ggtt->mappable_base, 2965 &ggtt->mappable_end); 2966 2967 ggtt->do_idle_maps = needs_idle_maps(dev_priv); 2968 ggtt->base.insert_page = i915_ggtt_insert_page; 2969 ggtt->base.insert_entries = i915_ggtt_insert_entries; 2970 ggtt->base.clear_range = i915_ggtt_clear_range; 2971 ggtt->base.bind_vma = ggtt_bind_vma; 2972 ggtt->base.unbind_vma = ggtt_unbind_vma; 2973 ggtt->base.cleanup = i915_gmch_remove; 2974 2975 ggtt->invalidate = gmch_ggtt_invalidate; 2976 2977 if (unlikely(ggtt->do_idle_maps)) 2978 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 2979 2980 return 0; 2981 } 2982 2983 /** 2984 * i915_ggtt_probe_hw - Probe GGTT hardware location 2985 * @dev_priv: i915 device 2986 */ 2987 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv) 2988 { 2989 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2990 int ret; 2991 2992 ggtt->base.i915 = dev_priv; 2993 ggtt->base.dma = &dev_priv->drm.pdev->dev; 2994 2995 if (INTEL_GEN(dev_priv) <= 5) 2996 ret = i915_gmch_probe(ggtt); 2997 else if (INTEL_GEN(dev_priv) < 8) 2998 ret = gen6_gmch_probe(ggtt); 2999 else 3000 ret = gen8_gmch_probe(ggtt); 3001 if (ret) 3002 return ret; 3003 3004 /* Trim the GGTT to fit the GuC mappable upper range (when enabled). 3005 * This is easier than doing range restriction on the fly, as we 3006 * currently don't have any bits spare to pass in this upper 3007 * restriction! 3008 */ 3009 if (HAS_GUC(dev_priv) && i915.enable_guc_loading) { 3010 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP); 3011 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total); 3012 } 3013 3014 if ((ggtt->base.total - 1) >> 32) { 3015 DRM_ERROR("We never expected a Global GTT with more than 32bits" 3016 " of address space! Found %lldM!\n", 3017 ggtt->base.total >> 20); 3018 ggtt->base.total = 1ULL << 32; 3019 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total); 3020 } 3021 3022 if (ggtt->mappable_end > ggtt->base.total) { 3023 DRM_ERROR("mappable aperture extends past end of GGTT," 3024 " aperture=%llx, total=%llx\n", 3025 ggtt->mappable_end, ggtt->base.total); 3026 ggtt->mappable_end = ggtt->base.total; 3027 } 3028 3029 /* GMADR is the PCI mmio aperture into the global GTT. */ 3030 DRM_INFO("Memory usable by graphics device = %lluM\n", 3031 ggtt->base.total >> 20); 3032 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20); 3033 DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20); 3034 #ifdef CONFIG_INTEL_IOMMU 3035 if (intel_iommu_gfx_mapped) 3036 DRM_INFO("VT-d active for gfx access\n"); 3037 #endif 3038 3039 return 0; 3040 } 3041 3042 /** 3043 * i915_ggtt_init_hw - Initialize GGTT hardware 3044 * @dev_priv: i915 device 3045 */ 3046 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) 3047 { 3048 struct i915_ggtt *ggtt = &dev_priv->ggtt; 3049 int ret; 3050 3051 INIT_LIST_HEAD(&dev_priv->vm_list); 3052 3053 /* Note that we use page colouring to enforce a guard page at the 3054 * end of the address space. This is required as the CS may prefetch 3055 * beyond the end of the batch buffer, across the page boundary, 3056 * and beyond the end of the GTT if we do not provide a guard. 3057 */ 3058 mutex_lock(&dev_priv->drm.struct_mutex); 3059 i915_address_space_init(&ggtt->base, dev_priv, "[global]"); 3060 if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv)) 3061 ggtt->base.mm.color_adjust = i915_gtt_color_adjust; 3062 mutex_unlock(&dev_priv->drm.struct_mutex); 3063 3064 if (!io_mapping_init_wc(&dev_priv->ggtt.mappable, 3065 dev_priv->ggtt.mappable_base, 3066 dev_priv->ggtt.mappable_end)) { 3067 ret = -EIO; 3068 goto out_gtt_cleanup; 3069 } 3070 3071 ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end); 3072 3073 /* 3074 * Initialise stolen early so that we may reserve preallocated 3075 * objects for the BIOS to KMS transition. 3076 */ 3077 ret = i915_gem_init_stolen(dev_priv); 3078 if (ret) 3079 goto out_gtt_cleanup; 3080 3081 return 0; 3082 3083 out_gtt_cleanup: 3084 ggtt->base.cleanup(&ggtt->base); 3085 return ret; 3086 } 3087 3088 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv) 3089 { 3090 if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt()) 3091 return -EIO; 3092 3093 return 0; 3094 } 3095 3096 void i915_ggtt_enable_guc(struct drm_i915_private *i915) 3097 { 3098 i915->ggtt.invalidate = guc_ggtt_invalidate; 3099 } 3100 3101 void i915_ggtt_disable_guc(struct drm_i915_private *i915) 3102 { 3103 if (i915->ggtt.invalidate == guc_ggtt_invalidate) 3104 i915->ggtt.invalidate = gen6_ggtt_invalidate; 3105 } 3106 3107 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) 3108 { 3109 struct i915_ggtt *ggtt = &dev_priv->ggtt; 3110 struct drm_i915_gem_object *obj, *on; 3111 3112 i915_check_and_clear_faults(dev_priv); 3113 3114 /* First fill our portion of the GTT with scratch pages */ 3115 ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total); 3116 3117 ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */ 3118 3119 /* clflush objects bound into the GGTT and rebind them. */ 3120 list_for_each_entry_safe(obj, on, 3121 &dev_priv->mm.bound_list, global_link) { 3122 bool ggtt_bound = false; 3123 struct i915_vma *vma; 3124 3125 list_for_each_entry(vma, &obj->vma_list, obj_link) { 3126 if (vma->vm != &ggtt->base) 3127 continue; 3128 3129 if (!i915_vma_unbind(vma)) 3130 continue; 3131 3132 WARN_ON(i915_vma_bind(vma, obj->cache_level, 3133 PIN_UPDATE)); 3134 ggtt_bound = true; 3135 } 3136 3137 if (ggtt_bound) 3138 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); 3139 } 3140 3141 ggtt->base.closed = false; 3142 3143 if (INTEL_GEN(dev_priv) >= 8) { 3144 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv)) 3145 chv_setup_private_ppat(dev_priv); 3146 else 3147 bdw_setup_private_ppat(dev_priv); 3148 3149 return; 3150 } 3151 3152 if (USES_PPGTT(dev_priv)) { 3153 struct i915_address_space *vm; 3154 3155 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3156 struct i915_hw_ppgtt *ppgtt; 3157 3158 if (i915_is_ggtt(vm)) 3159 ppgtt = dev_priv->mm.aliasing_ppgtt; 3160 else 3161 ppgtt = i915_vm_to_ppgtt(vm); 3162 3163 gen6_write_page_range(ppgtt, 0, ppgtt->base.total); 3164 } 3165 } 3166 3167 i915_ggtt_invalidate(dev_priv); 3168 } 3169 3170 static struct scatterlist * 3171 rotate_pages(const dma_addr_t *in, unsigned int offset, 3172 unsigned int width, unsigned int height, 3173 unsigned int stride, 3174 struct sg_table *st, struct scatterlist *sg) 3175 { 3176 unsigned int column, row; 3177 unsigned int src_idx; 3178 3179 for (column = 0; column < width; column++) { 3180 src_idx = stride * (height - 1) + column; 3181 for (row = 0; row < height; row++) { 3182 st->nents++; 3183 /* We don't need the pages, but need to initialize 3184 * the entries so the sg list can be happily traversed. 3185 * The only thing we need are DMA addresses. 3186 */ 3187 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3188 sg_dma_address(sg) = in[offset + src_idx]; 3189 sg_dma_len(sg) = PAGE_SIZE; 3190 sg = sg_next(sg); 3191 src_idx -= stride; 3192 } 3193 } 3194 3195 return sg; 3196 } 3197 3198 static noinline struct sg_table * 3199 intel_rotate_pages(struct intel_rotation_info *rot_info, 3200 struct drm_i915_gem_object *obj) 3201 { 3202 const unsigned long n_pages = obj->base.size / PAGE_SIZE; 3203 unsigned int size = intel_rotation_info_size(rot_info); 3204 struct sgt_iter sgt_iter; 3205 dma_addr_t dma_addr; 3206 unsigned long i; 3207 dma_addr_t *page_addr_list; 3208 struct sg_table *st; 3209 struct scatterlist *sg; 3210 int ret = -ENOMEM; 3211 3212 /* Allocate a temporary list of source pages for random access. */ 3213 page_addr_list = drm_malloc_gfp(n_pages, 3214 sizeof(dma_addr_t), 3215 GFP_TEMPORARY); 3216 if (!page_addr_list) 3217 return ERR_PTR(ret); 3218 3219 /* Allocate target SG list. */ 3220 st = kmalloc(sizeof(*st), GFP_KERNEL); 3221 if (!st) 3222 goto err_st_alloc; 3223 3224 ret = sg_alloc_table(st, size, GFP_KERNEL); 3225 if (ret) 3226 goto err_sg_alloc; 3227 3228 /* Populate source page list from the object. */ 3229 i = 0; 3230 for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) 3231 page_addr_list[i++] = dma_addr; 3232 3233 GEM_BUG_ON(i != n_pages); 3234 st->nents = 0; 3235 sg = st->sgl; 3236 3237 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { 3238 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset, 3239 rot_info->plane[i].width, rot_info->plane[i].height, 3240 rot_info->plane[i].stride, st, sg); 3241 } 3242 3243 DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n", 3244 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); 3245 3246 drm_free_large(page_addr_list); 3247 3248 return st; 3249 3250 err_sg_alloc: 3251 kfree(st); 3252 err_st_alloc: 3253 drm_free_large(page_addr_list); 3254 3255 DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 3256 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); 3257 3258 return ERR_PTR(ret); 3259 } 3260 3261 static noinline struct sg_table * 3262 intel_partial_pages(const struct i915_ggtt_view *view, 3263 struct drm_i915_gem_object *obj) 3264 { 3265 struct sg_table *st; 3266 struct scatterlist *sg, *iter; 3267 unsigned int count = view->partial.size; 3268 unsigned int offset; 3269 int ret = -ENOMEM; 3270 3271 st = kmalloc(sizeof(*st), GFP_KERNEL); 3272 if (!st) 3273 goto err_st_alloc; 3274 3275 ret = sg_alloc_table(st, count, GFP_KERNEL); 3276 if (ret) 3277 goto err_sg_alloc; 3278 3279 iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); 3280 GEM_BUG_ON(!iter); 3281 3282 sg = st->sgl; 3283 st->nents = 0; 3284 do { 3285 unsigned int len; 3286 3287 len = min(iter->length - (offset << PAGE_SHIFT), 3288 count << PAGE_SHIFT); 3289 sg_set_page(sg, NULL, len, 0); 3290 sg_dma_address(sg) = 3291 sg_dma_address(iter) + (offset << PAGE_SHIFT); 3292 sg_dma_len(sg) = len; 3293 3294 st->nents++; 3295 count -= len >> PAGE_SHIFT; 3296 if (count == 0) { 3297 sg_mark_end(sg); 3298 return st; 3299 } 3300 3301 sg = __sg_next(sg); 3302 iter = __sg_next(iter); 3303 offset = 0; 3304 } while (1); 3305 3306 err_sg_alloc: 3307 kfree(st); 3308 err_st_alloc: 3309 return ERR_PTR(ret); 3310 } 3311 3312 static int 3313 i915_get_ggtt_vma_pages(struct i915_vma *vma) 3314 { 3315 int ret; 3316 3317 /* The vma->pages are only valid within the lifespan of the borrowed 3318 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 3319 * must be the vma->pages. A simple rule is that vma->pages must only 3320 * be accessed when the obj->mm.pages are pinned. 3321 */ 3322 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 3323 3324 switch (vma->ggtt_view.type) { 3325 case I915_GGTT_VIEW_NORMAL: 3326 vma->pages = vma->obj->mm.pages; 3327 return 0; 3328 3329 case I915_GGTT_VIEW_ROTATED: 3330 vma->pages = 3331 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); 3332 break; 3333 3334 case I915_GGTT_VIEW_PARTIAL: 3335 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); 3336 break; 3337 3338 default: 3339 WARN_ONCE(1, "GGTT view %u not implemented!\n", 3340 vma->ggtt_view.type); 3341 return -EINVAL; 3342 } 3343 3344 ret = 0; 3345 if (unlikely(IS_ERR(vma->pages))) { 3346 ret = PTR_ERR(vma->pages); 3347 vma->pages = NULL; 3348 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 3349 vma->ggtt_view.type, ret); 3350 } 3351 return ret; 3352 } 3353 3354 /** 3355 * i915_gem_gtt_reserve - reserve a node in an address_space (GTT) 3356 * @vm: the &struct i915_address_space 3357 * @node: the &struct drm_mm_node (typically i915_vma.mode) 3358 * @size: how much space to allocate inside the GTT, 3359 * must be #I915_GTT_PAGE_SIZE aligned 3360 * @offset: where to insert inside the GTT, 3361 * must be #I915_GTT_MIN_ALIGNMENT aligned, and the node 3362 * (@offset + @size) must fit within the address space 3363 * @color: color to apply to node, if this node is not from a VMA, 3364 * color must be #I915_COLOR_UNEVICTABLE 3365 * @flags: control search and eviction behaviour 3366 * 3367 * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside 3368 * the address space (using @size and @color). If the @node does not fit, it 3369 * tries to evict any overlapping nodes from the GTT, including any 3370 * neighbouring nodes if the colors do not match (to ensure guard pages between 3371 * differing domains). See i915_gem_evict_for_node() for the gory details 3372 * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on 3373 * evicting active overlapping objects, and any overlapping node that is pinned 3374 * or marked as unevictable will also result in failure. 3375 * 3376 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if 3377 * asked to wait for eviction and interrupted. 3378 */ 3379 int i915_gem_gtt_reserve(struct i915_address_space *vm, 3380 struct drm_mm_node *node, 3381 u64 size, u64 offset, unsigned long color, 3382 unsigned int flags) 3383 { 3384 int err; 3385 3386 GEM_BUG_ON(!size); 3387 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 3388 GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT)); 3389 GEM_BUG_ON(range_overflows(offset, size, vm->total)); 3390 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base); 3391 GEM_BUG_ON(drm_mm_node_allocated(node)); 3392 3393 node->size = size; 3394 node->start = offset; 3395 node->color = color; 3396 3397 err = drm_mm_reserve_node(&vm->mm, node); 3398 if (err != -ENOSPC) 3399 return err; 3400 3401 err = i915_gem_evict_for_node(vm, node, flags); 3402 if (err == 0) 3403 err = drm_mm_reserve_node(&vm->mm, node); 3404 3405 return err; 3406 } 3407 3408 static u64 random_offset(u64 start, u64 end, u64 len, u64 align) 3409 { 3410 u64 range, addr; 3411 3412 GEM_BUG_ON(range_overflows(start, len, end)); 3413 GEM_BUG_ON(round_up(start, align) > round_down(end - len, align)); 3414 3415 range = round_down(end - len, align) - round_up(start, align); 3416 if (range) { 3417 if (sizeof(unsigned long) == sizeof(u64)) { 3418 addr = get_random_long(); 3419 } else { 3420 addr = get_random_int(); 3421 if (range > U32_MAX) { 3422 addr <<= 32; 3423 addr |= get_random_int(); 3424 } 3425 } 3426 div64_u64_rem(addr, range, &addr); 3427 start += addr; 3428 } 3429 3430 return round_up(start, align); 3431 } 3432 3433 /** 3434 * i915_gem_gtt_insert - insert a node into an address_space (GTT) 3435 * @vm: the &struct i915_address_space 3436 * @node: the &struct drm_mm_node (typically i915_vma.node) 3437 * @size: how much space to allocate inside the GTT, 3438 * must be #I915_GTT_PAGE_SIZE aligned 3439 * @alignment: required alignment of starting offset, may be 0 but 3440 * if specified, this must be a power-of-two and at least 3441 * #I915_GTT_MIN_ALIGNMENT 3442 * @color: color to apply to node 3443 * @start: start of any range restriction inside GTT (0 for all), 3444 * must be #I915_GTT_PAGE_SIZE aligned 3445 * @end: end of any range restriction inside GTT (U64_MAX for all), 3446 * must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX 3447 * @flags: control search and eviction behaviour 3448 * 3449 * i915_gem_gtt_insert() first searches for an available hole into which 3450 * is can insert the node. The hole address is aligned to @alignment and 3451 * its @size must then fit entirely within the [@start, @end] bounds. The 3452 * nodes on either side of the hole must match @color, or else a guard page 3453 * will be inserted between the two nodes (or the node evicted). If no 3454 * suitable hole is found, first a victim is randomly selected and tested 3455 * for eviction, otherwise then the LRU list of objects within the GTT 3456 * is scanned to find the first set of replacement nodes to create the hole. 3457 * Those old overlapping nodes are evicted from the GTT (and so must be 3458 * rebound before any future use). Any node that is currently pinned cannot 3459 * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently 3460 * active and #PIN_NONBLOCK is specified, that node is also skipped when 3461 * searching for an eviction candidate. See i915_gem_evict_something() for 3462 * the gory details on the eviction algorithm. 3463 * 3464 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if 3465 * asked to wait for eviction and interrupted. 3466 */ 3467 int i915_gem_gtt_insert(struct i915_address_space *vm, 3468 struct drm_mm_node *node, 3469 u64 size, u64 alignment, unsigned long color, 3470 u64 start, u64 end, unsigned int flags) 3471 { 3472 enum drm_mm_insert_mode mode; 3473 u64 offset; 3474 int err; 3475 3476 lockdep_assert_held(&vm->i915->drm.struct_mutex); 3477 GEM_BUG_ON(!size); 3478 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 3479 GEM_BUG_ON(alignment && !is_power_of_2(alignment)); 3480 GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); 3481 GEM_BUG_ON(start >= end); 3482 GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 3483 GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 3484 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base); 3485 GEM_BUG_ON(drm_mm_node_allocated(node)); 3486 3487 if (unlikely(range_overflows(start, size, end))) 3488 return -ENOSPC; 3489 3490 if (unlikely(round_up(start, alignment) > round_down(end - size, alignment))) 3491 return -ENOSPC; 3492 3493 mode = DRM_MM_INSERT_BEST; 3494 if (flags & PIN_HIGH) 3495 mode = DRM_MM_INSERT_HIGH; 3496 if (flags & PIN_MAPPABLE) 3497 mode = DRM_MM_INSERT_LOW; 3498 3499 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, 3500 * so we know that we always have a minimum alignment of 4096. 3501 * The drm_mm range manager is optimised to return results 3502 * with zero alignment, so where possible use the optimal 3503 * path. 3504 */ 3505 BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE); 3506 if (alignment <= I915_GTT_MIN_ALIGNMENT) 3507 alignment = 0; 3508 3509 err = drm_mm_insert_node_in_range(&vm->mm, node, 3510 size, alignment, color, 3511 start, end, mode); 3512 if (err != -ENOSPC) 3513 return err; 3514 3515 /* No free space, pick a slot at random. 3516 * 3517 * There is a pathological case here using a GTT shared between 3518 * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt): 3519 * 3520 * |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->| 3521 * (64k objects) (448k objects) 3522 * 3523 * Now imagine that the eviction LRU is ordered top-down (just because 3524 * pathology meets real life), and that we need to evict an object to 3525 * make room inside the aperture. The eviction scan then has to walk 3526 * the 448k list before it finds one within range. And now imagine that 3527 * it has to search for a new hole between every byte inside the memcpy, 3528 * for several simultaneous clients. 3529 * 3530 * On a full-ppgtt system, if we have run out of available space, there 3531 * will be lots and lots of objects in the eviction list! Again, 3532 * searching that LRU list may be slow if we are also applying any 3533 * range restrictions (e.g. restriction to low 4GiB) and so, for 3534 * simplicity and similarilty between different GTT, try the single 3535 * random replacement first. 3536 */ 3537 offset = random_offset(start, end, 3538 size, alignment ?: I915_GTT_MIN_ALIGNMENT); 3539 err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags); 3540 if (err != -ENOSPC) 3541 return err; 3542 3543 /* Randomly selected placement is pinned, do a search */ 3544 err = i915_gem_evict_something(vm, size, alignment, color, 3545 start, end, flags); 3546 if (err) 3547 return err; 3548 3549 return drm_mm_insert_node_in_range(&vm->mm, node, 3550 size, alignment, color, 3551 start, end, DRM_MM_INSERT_EVICT); 3552 } 3553 3554 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 3555 #include "selftests/mock_gtt.c" 3556 #include "selftests/i915_gem_gtt.c" 3557 #endif 3558