1 /* 2 * Copyright © 2010 Daniel Vetter 3 * Copyright © 2011-2014 Intel Corporation 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/seq_file.h> 27 #include <drm/drmP.h> 28 #include <drm/i915_drm.h> 29 #include "i915_drv.h" 30 #include "i915_vgpu.h" 31 #include "i915_trace.h" 32 #include "intel_drv.h" 33 34 /** 35 * DOC: Global GTT views 36 * 37 * Background and previous state 38 * 39 * Historically objects could exists (be bound) in global GTT space only as 40 * singular instances with a view representing all of the object's backing pages 41 * in a linear fashion. This view will be called a normal view. 42 * 43 * To support multiple views of the same object, where the number of mapped 44 * pages is not equal to the backing store, or where the layout of the pages 45 * is not linear, concept of a GGTT view was added. 46 * 47 * One example of an alternative view is a stereo display driven by a single 48 * image. In this case we would have a framebuffer looking like this 49 * (2x2 pages): 50 * 51 * 12 52 * 34 53 * 54 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 55 * rendering. In contrast, fed to the display engine would be an alternative 56 * view which could look something like this: 57 * 58 * 1212 59 * 3434 60 * 61 * In this example both the size and layout of pages in the alternative view is 62 * different from the normal view. 63 * 64 * Implementation and usage 65 * 66 * GGTT views are implemented using VMAs and are distinguished via enum 67 * i915_ggtt_view_type and struct i915_ggtt_view. 68 * 69 * A new flavour of core GEM functions which work with GGTT bound objects were 70 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 71 * renaming in large amounts of code. They take the struct i915_ggtt_view 72 * parameter encapsulating all metadata required to implement a view. 73 * 74 * As a helper for callers which are only interested in the normal view, 75 * globally const i915_ggtt_view_normal singleton instance exists. All old core 76 * GEM API functions, the ones not taking the view parameter, are operating on, 77 * or with the normal GGTT view. 78 * 79 * Code wanting to add or use a new GGTT view needs to: 80 * 81 * 1. Add a new enum with a suitable name. 82 * 2. Extend the metadata in the i915_ggtt_view structure if required. 83 * 3. Add support to i915_get_vma_pages(). 84 * 85 * New views are required to build a scatter-gather table from within the 86 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 87 * exists for the lifetime of an VMA. 88 * 89 * Core API is designed to have copy semantics which means that passed in 90 * struct i915_ggtt_view does not need to be persistent (left around after 91 * calling the core API functions). 92 * 93 */ 94 95 static int 96 i915_get_ggtt_vma_pages(struct i915_vma *vma); 97 98 const struct i915_ggtt_view i915_ggtt_view_normal; 99 const struct i915_ggtt_view i915_ggtt_view_rotated = { 100 .type = I915_GGTT_VIEW_ROTATED 101 }; 102 103 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) 104 { 105 bool has_aliasing_ppgtt; 106 bool has_full_ppgtt; 107 108 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; 109 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; 110 111 if (intel_vgpu_active(dev)) 112 has_full_ppgtt = false; /* emulation is too hard */ 113 114 /* 115 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 116 * execlists, the sole mechanism available to submit work. 117 */ 118 if (INTEL_INFO(dev)->gen < 9 && 119 (enable_ppgtt == 0 || !has_aliasing_ppgtt)) 120 return 0; 121 122 if (enable_ppgtt == 1) 123 return 1; 124 125 if (enable_ppgtt == 2 && has_full_ppgtt) 126 return 2; 127 128 #ifdef CONFIG_INTEL_IOMMU 129 /* Disable ppgtt on SNB if VT-d is on. */ 130 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { 131 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 132 return 0; 133 } 134 #endif 135 136 /* Early VLV doesn't have this */ 137 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && 138 dev->pdev->revision < 0xb) { 139 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 140 return 0; 141 } 142 143 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) 144 return 2; 145 else 146 return has_aliasing_ppgtt ? 1 : 0; 147 } 148 149 static int ppgtt_bind_vma(struct i915_vma *vma, 150 enum i915_cache_level cache_level, 151 u32 unused) 152 { 153 u32 pte_flags = 0; 154 155 /* Currently applicable only to VLV */ 156 if (vma->obj->gt_ro) 157 pte_flags |= PTE_READ_ONLY; 158 159 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, 160 cache_level, pte_flags); 161 162 return 0; 163 } 164 165 static void ppgtt_unbind_vma(struct i915_vma *vma) 166 { 167 vma->vm->clear_range(vma->vm, 168 vma->node.start, 169 vma->obj->base.size, 170 true); 171 } 172 173 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 174 enum i915_cache_level level, 175 bool valid) 176 { 177 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; 178 pte |= addr; 179 180 switch (level) { 181 case I915_CACHE_NONE: 182 pte |= PPAT_UNCACHED_INDEX; 183 break; 184 case I915_CACHE_WT: 185 pte |= PPAT_DISPLAY_ELLC_INDEX; 186 break; 187 default: 188 pte |= PPAT_CACHED_INDEX; 189 break; 190 } 191 192 return pte; 193 } 194 195 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 196 const enum i915_cache_level level) 197 { 198 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 199 pde |= addr; 200 if (level != I915_CACHE_NONE) 201 pde |= PPAT_CACHED_PDE_INDEX; 202 else 203 pde |= PPAT_UNCACHED_INDEX; 204 return pde; 205 } 206 207 #define gen8_pdpe_encode gen8_pde_encode 208 #define gen8_pml4e_encode gen8_pde_encode 209 210 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 211 enum i915_cache_level level, 212 bool valid, u32 unused) 213 { 214 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 215 pte |= GEN6_PTE_ADDR_ENCODE(addr); 216 217 switch (level) { 218 case I915_CACHE_L3_LLC: 219 case I915_CACHE_LLC: 220 pte |= GEN6_PTE_CACHE_LLC; 221 break; 222 case I915_CACHE_NONE: 223 pte |= GEN6_PTE_UNCACHED; 224 break; 225 default: 226 MISSING_CASE(level); 227 } 228 229 return pte; 230 } 231 232 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 233 enum i915_cache_level level, 234 bool valid, u32 unused) 235 { 236 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 237 pte |= GEN6_PTE_ADDR_ENCODE(addr); 238 239 switch (level) { 240 case I915_CACHE_L3_LLC: 241 pte |= GEN7_PTE_CACHE_L3_LLC; 242 break; 243 case I915_CACHE_LLC: 244 pte |= GEN6_PTE_CACHE_LLC; 245 break; 246 case I915_CACHE_NONE: 247 pte |= GEN6_PTE_UNCACHED; 248 break; 249 default: 250 MISSING_CASE(level); 251 } 252 253 return pte; 254 } 255 256 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 257 enum i915_cache_level level, 258 bool valid, u32 flags) 259 { 260 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 261 pte |= GEN6_PTE_ADDR_ENCODE(addr); 262 263 if (!(flags & PTE_READ_ONLY)) 264 pte |= BYT_PTE_WRITEABLE; 265 266 if (level != I915_CACHE_NONE) 267 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 268 269 return pte; 270 } 271 272 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 273 enum i915_cache_level level, 274 bool valid, u32 unused) 275 { 276 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 277 pte |= HSW_PTE_ADDR_ENCODE(addr); 278 279 if (level != I915_CACHE_NONE) 280 pte |= HSW_WB_LLC_AGE3; 281 282 return pte; 283 } 284 285 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 286 enum i915_cache_level level, 287 bool valid, u32 unused) 288 { 289 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 290 pte |= HSW_PTE_ADDR_ENCODE(addr); 291 292 switch (level) { 293 case I915_CACHE_NONE: 294 break; 295 case I915_CACHE_WT: 296 pte |= HSW_WT_ELLC_LLC_AGE3; 297 break; 298 default: 299 pte |= HSW_WB_ELLC_LLC_AGE3; 300 break; 301 } 302 303 return pte; 304 } 305 306 static int __setup_page_dma(struct drm_device *dev, 307 struct i915_page_dma *p, gfp_t flags) 308 { 309 struct device *device = &dev->pdev->dev; 310 311 p->page = alloc_page(flags); 312 if (!p->page) 313 return -ENOMEM; 314 315 p->daddr = dma_map_page(device, 316 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); 317 318 if (dma_mapping_error(device, p->daddr)) { 319 __free_page(p->page); 320 return -EINVAL; 321 } 322 323 return 0; 324 } 325 326 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 327 { 328 return __setup_page_dma(dev, p, GFP_KERNEL); 329 } 330 331 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 332 { 333 if (WARN_ON(!p->page)) 334 return; 335 336 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); 337 __free_page(p->page); 338 memset(p, 0, sizeof(*p)); 339 } 340 341 static void *kmap_page_dma(struct i915_page_dma *p) 342 { 343 return kmap_atomic(p->page); 344 } 345 346 /* We use the flushing unmap only with ppgtt structures: 347 * page directories, page tables and scratch pages. 348 */ 349 static void kunmap_page_dma(struct drm_device *dev, void *vaddr) 350 { 351 /* There are only few exceptions for gen >=6. chv and bxt. 352 * And we are not sure about the latter so play safe for now. 353 */ 354 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 355 drm_clflush_virt_range(vaddr, PAGE_SIZE); 356 357 kunmap_atomic(vaddr); 358 } 359 360 #define kmap_px(px) kmap_page_dma(px_base(px)) 361 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) 362 363 #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 364 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 365 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) 366 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) 367 368 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, 369 const uint64_t val) 370 { 371 int i; 372 uint64_t * const vaddr = kmap_page_dma(p); 373 374 for (i = 0; i < 512; i++) 375 vaddr[i] = val; 376 377 kunmap_page_dma(dev, vaddr); 378 } 379 380 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, 381 const uint32_t val32) 382 { 383 uint64_t v = val32; 384 385 v = v << 32 | val32; 386 387 fill_page_dma(dev, p, v); 388 } 389 390 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) 391 { 392 struct i915_page_scratch *sp; 393 int ret; 394 395 sp = kzalloc(sizeof(*sp), GFP_KERNEL); 396 if (sp == NULL) 397 return ERR_PTR(-ENOMEM); 398 399 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); 400 if (ret) { 401 kfree(sp); 402 return ERR_PTR(ret); 403 } 404 405 set_pages_uc(px_page(sp), 1); 406 407 return sp; 408 } 409 410 static void free_scratch_page(struct drm_device *dev, 411 struct i915_page_scratch *sp) 412 { 413 set_pages_wb(px_page(sp), 1); 414 415 cleanup_px(dev, sp); 416 kfree(sp); 417 } 418 419 static struct i915_page_table *alloc_pt(struct drm_device *dev) 420 { 421 struct i915_page_table *pt; 422 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 423 GEN8_PTES : GEN6_PTES; 424 int ret = -ENOMEM; 425 426 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 427 if (!pt) 428 return ERR_PTR(-ENOMEM); 429 430 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), 431 GFP_KERNEL); 432 433 if (!pt->used_ptes) 434 goto fail_bitmap; 435 436 ret = setup_px(dev, pt); 437 if (ret) 438 goto fail_page_m; 439 440 return pt; 441 442 fail_page_m: 443 kfree(pt->used_ptes); 444 fail_bitmap: 445 kfree(pt); 446 447 return ERR_PTR(ret); 448 } 449 450 static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 451 { 452 cleanup_px(dev, pt); 453 kfree(pt->used_ptes); 454 kfree(pt); 455 } 456 457 static void gen8_initialize_pt(struct i915_address_space *vm, 458 struct i915_page_table *pt) 459 { 460 gen8_pte_t scratch_pte; 461 462 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 463 I915_CACHE_LLC, true); 464 465 fill_px(vm->dev, pt, scratch_pte); 466 } 467 468 static void gen6_initialize_pt(struct i915_address_space *vm, 469 struct i915_page_table *pt) 470 { 471 gen6_pte_t scratch_pte; 472 473 WARN_ON(px_dma(vm->scratch_page) == 0); 474 475 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 476 I915_CACHE_LLC, true, 0); 477 478 fill32_px(vm->dev, pt, scratch_pte); 479 } 480 481 static struct i915_page_directory *alloc_pd(struct drm_device *dev) 482 { 483 struct i915_page_directory *pd; 484 int ret = -ENOMEM; 485 486 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 487 if (!pd) 488 return ERR_PTR(-ENOMEM); 489 490 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), 491 sizeof(*pd->used_pdes), GFP_KERNEL); 492 if (!pd->used_pdes) 493 goto fail_bitmap; 494 495 ret = setup_px(dev, pd); 496 if (ret) 497 goto fail_page_m; 498 499 return pd; 500 501 fail_page_m: 502 kfree(pd->used_pdes); 503 fail_bitmap: 504 kfree(pd); 505 506 return ERR_PTR(ret); 507 } 508 509 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 510 { 511 if (px_page(pd)) { 512 cleanup_px(dev, pd); 513 kfree(pd->used_pdes); 514 kfree(pd); 515 } 516 } 517 518 static void gen8_initialize_pd(struct i915_address_space *vm, 519 struct i915_page_directory *pd) 520 { 521 gen8_pde_t scratch_pde; 522 523 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); 524 525 fill_px(vm->dev, pd, scratch_pde); 526 } 527 528 static int __pdp_init(struct drm_device *dev, 529 struct i915_page_directory_pointer *pdp) 530 { 531 size_t pdpes = I915_PDPES_PER_PDP(dev); 532 533 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), 534 sizeof(unsigned long), 535 GFP_KERNEL); 536 if (!pdp->used_pdpes) 537 return -ENOMEM; 538 539 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), 540 GFP_KERNEL); 541 if (!pdp->page_directory) { 542 kfree(pdp->used_pdpes); 543 /* the PDP might be the statically allocated top level. Keep it 544 * as clean as possible */ 545 pdp->used_pdpes = NULL; 546 return -ENOMEM; 547 } 548 549 return 0; 550 } 551 552 static void __pdp_fini(struct i915_page_directory_pointer *pdp) 553 { 554 kfree(pdp->used_pdpes); 555 kfree(pdp->page_directory); 556 pdp->page_directory = NULL; 557 } 558 559 static struct 560 i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) 561 { 562 struct i915_page_directory_pointer *pdp; 563 int ret = -ENOMEM; 564 565 WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); 566 567 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 568 if (!pdp) 569 return ERR_PTR(-ENOMEM); 570 571 ret = __pdp_init(dev, pdp); 572 if (ret) 573 goto fail_bitmap; 574 575 ret = setup_px(dev, pdp); 576 if (ret) 577 goto fail_page_m; 578 579 return pdp; 580 581 fail_page_m: 582 __pdp_fini(pdp); 583 fail_bitmap: 584 kfree(pdp); 585 586 return ERR_PTR(ret); 587 } 588 589 static void free_pdp(struct drm_device *dev, 590 struct i915_page_directory_pointer *pdp) 591 { 592 __pdp_fini(pdp); 593 if (USES_FULL_48BIT_PPGTT(dev)) { 594 cleanup_px(dev, pdp); 595 kfree(pdp); 596 } 597 } 598 599 static void gen8_initialize_pdp(struct i915_address_space *vm, 600 struct i915_page_directory_pointer *pdp) 601 { 602 gen8_ppgtt_pdpe_t scratch_pdpe; 603 604 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); 605 606 fill_px(vm->dev, pdp, scratch_pdpe); 607 } 608 609 static void gen8_initialize_pml4(struct i915_address_space *vm, 610 struct i915_pml4 *pml4) 611 { 612 gen8_ppgtt_pml4e_t scratch_pml4e; 613 614 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), 615 I915_CACHE_LLC); 616 617 fill_px(vm->dev, pml4, scratch_pml4e); 618 } 619 620 static void 621 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, 622 struct i915_page_directory_pointer *pdp, 623 struct i915_page_directory *pd, 624 int index) 625 { 626 gen8_ppgtt_pdpe_t *page_directorypo; 627 628 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 629 return; 630 631 page_directorypo = kmap_px(pdp); 632 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); 633 kunmap_px(ppgtt, page_directorypo); 634 } 635 636 static void 637 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, 638 struct i915_pml4 *pml4, 639 struct i915_page_directory_pointer *pdp, 640 int index) 641 { 642 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); 643 644 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); 645 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 646 kunmap_px(ppgtt, pagemap); 647 } 648 649 /* Broadwell Page Directory Pointer Descriptors */ 650 static int gen8_write_pdp(struct drm_i915_gem_request *req, 651 unsigned entry, 652 dma_addr_t addr) 653 { 654 struct intel_engine_cs *ring = req->ring; 655 int ret; 656 657 BUG_ON(entry >= 4); 658 659 ret = intel_ring_begin(req, 6); 660 if (ret) 661 return ret; 662 663 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 664 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); 665 intel_ring_emit(ring, upper_32_bits(addr)); 666 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 667 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); 668 intel_ring_emit(ring, lower_32_bits(addr)); 669 intel_ring_advance(ring); 670 671 return 0; 672 } 673 674 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, 675 struct drm_i915_gem_request *req) 676 { 677 int i, ret; 678 679 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { 680 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 681 682 ret = gen8_write_pdp(req, i, pd_daddr); 683 if (ret) 684 return ret; 685 } 686 687 return 0; 688 } 689 690 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, 691 struct drm_i915_gem_request *req) 692 { 693 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 694 } 695 696 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, 697 struct i915_page_directory_pointer *pdp, 698 uint64_t start, 699 uint64_t length, 700 gen8_pte_t scratch_pte) 701 { 702 struct i915_hw_ppgtt *ppgtt = 703 container_of(vm, struct i915_hw_ppgtt, base); 704 gen8_pte_t *pt_vaddr; 705 unsigned pdpe = gen8_pdpe_index(start); 706 unsigned pde = gen8_pde_index(start); 707 unsigned pte = gen8_pte_index(start); 708 unsigned num_entries = length >> PAGE_SHIFT; 709 unsigned last_pte, i; 710 711 if (WARN_ON(!pdp)) 712 return; 713 714 while (num_entries) { 715 struct i915_page_directory *pd; 716 struct i915_page_table *pt; 717 718 if (WARN_ON(!pdp->page_directory[pdpe])) 719 break; 720 721 pd = pdp->page_directory[pdpe]; 722 723 if (WARN_ON(!pd->page_table[pde])) 724 break; 725 726 pt = pd->page_table[pde]; 727 728 if (WARN_ON(!px_page(pt))) 729 break; 730 731 last_pte = pte + num_entries; 732 if (last_pte > GEN8_PTES) 733 last_pte = GEN8_PTES; 734 735 pt_vaddr = kmap_px(pt); 736 737 for (i = pte; i < last_pte; i++) { 738 pt_vaddr[i] = scratch_pte; 739 num_entries--; 740 } 741 742 kunmap_px(ppgtt, pt); 743 744 pte = 0; 745 if (++pde == I915_PDES) { 746 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 747 break; 748 pde = 0; 749 } 750 } 751 } 752 753 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, 754 uint64_t start, 755 uint64_t length, 756 bool use_scratch) 757 { 758 struct i915_hw_ppgtt *ppgtt = 759 container_of(vm, struct i915_hw_ppgtt, base); 760 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 761 I915_CACHE_LLC, use_scratch); 762 763 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 764 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, 765 scratch_pte); 766 } else { 767 uint64_t templ4, pml4e; 768 struct i915_page_directory_pointer *pdp; 769 770 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 771 gen8_ppgtt_clear_pte_range(vm, pdp, start, length, 772 scratch_pte); 773 } 774 } 775 } 776 777 static void 778 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, 779 struct i915_page_directory_pointer *pdp, 780 struct sg_page_iter *sg_iter, 781 uint64_t start, 782 enum i915_cache_level cache_level) 783 { 784 struct i915_hw_ppgtt *ppgtt = 785 container_of(vm, struct i915_hw_ppgtt, base); 786 gen8_pte_t *pt_vaddr; 787 unsigned pdpe = gen8_pdpe_index(start); 788 unsigned pde = gen8_pde_index(start); 789 unsigned pte = gen8_pte_index(start); 790 791 pt_vaddr = NULL; 792 793 while (__sg_page_iter_next(sg_iter)) { 794 if (pt_vaddr == NULL) { 795 struct i915_page_directory *pd = pdp->page_directory[pdpe]; 796 struct i915_page_table *pt = pd->page_table[pde]; 797 pt_vaddr = kmap_px(pt); 798 } 799 800 pt_vaddr[pte] = 801 gen8_pte_encode(sg_page_iter_dma_address(sg_iter), 802 cache_level, true); 803 if (++pte == GEN8_PTES) { 804 kunmap_px(ppgtt, pt_vaddr); 805 pt_vaddr = NULL; 806 if (++pde == I915_PDES) { 807 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 808 break; 809 pde = 0; 810 } 811 pte = 0; 812 } 813 } 814 815 if (pt_vaddr) 816 kunmap_px(ppgtt, pt_vaddr); 817 } 818 819 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 820 struct sg_table *pages, 821 uint64_t start, 822 enum i915_cache_level cache_level, 823 u32 unused) 824 { 825 struct i915_hw_ppgtt *ppgtt = 826 container_of(vm, struct i915_hw_ppgtt, base); 827 struct sg_page_iter sg_iter; 828 829 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); 830 831 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 832 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, 833 cache_level); 834 } else { 835 struct i915_page_directory_pointer *pdp; 836 uint64_t templ4, pml4e; 837 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; 838 839 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 840 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, 841 start, cache_level); 842 } 843 } 844 } 845 846 static void gen8_free_page_tables(struct drm_device *dev, 847 struct i915_page_directory *pd) 848 { 849 int i; 850 851 if (!px_page(pd)) 852 return; 853 854 for_each_set_bit(i, pd->used_pdes, I915_PDES) { 855 if (WARN_ON(!pd->page_table[i])) 856 continue; 857 858 free_pt(dev, pd->page_table[i]); 859 pd->page_table[i] = NULL; 860 } 861 } 862 863 static int gen8_init_scratch(struct i915_address_space *vm) 864 { 865 struct drm_device *dev = vm->dev; 866 867 vm->scratch_page = alloc_scratch_page(dev); 868 if (IS_ERR(vm->scratch_page)) 869 return PTR_ERR(vm->scratch_page); 870 871 vm->scratch_pt = alloc_pt(dev); 872 if (IS_ERR(vm->scratch_pt)) { 873 free_scratch_page(dev, vm->scratch_page); 874 return PTR_ERR(vm->scratch_pt); 875 } 876 877 vm->scratch_pd = alloc_pd(dev); 878 if (IS_ERR(vm->scratch_pd)) { 879 free_pt(dev, vm->scratch_pt); 880 free_scratch_page(dev, vm->scratch_page); 881 return PTR_ERR(vm->scratch_pd); 882 } 883 884 if (USES_FULL_48BIT_PPGTT(dev)) { 885 vm->scratch_pdp = alloc_pdp(dev); 886 if (IS_ERR(vm->scratch_pdp)) { 887 free_pd(dev, vm->scratch_pd); 888 free_pt(dev, vm->scratch_pt); 889 free_scratch_page(dev, vm->scratch_page); 890 return PTR_ERR(vm->scratch_pdp); 891 } 892 } 893 894 gen8_initialize_pt(vm, vm->scratch_pt); 895 gen8_initialize_pd(vm, vm->scratch_pd); 896 if (USES_FULL_48BIT_PPGTT(dev)) 897 gen8_initialize_pdp(vm, vm->scratch_pdp); 898 899 return 0; 900 } 901 902 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) 903 { 904 enum vgt_g2v_type msg; 905 struct drm_device *dev = ppgtt->base.dev; 906 struct drm_i915_private *dev_priv = dev->dev_private; 907 unsigned int offset = vgtif_reg(pdp0_lo); 908 int i; 909 910 if (USES_FULL_48BIT_PPGTT(dev)) { 911 u64 daddr = px_dma(&ppgtt->pml4); 912 913 I915_WRITE(offset, lower_32_bits(daddr)); 914 I915_WRITE(offset + 4, upper_32_bits(daddr)); 915 916 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : 917 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); 918 } else { 919 for (i = 0; i < GEN8_LEGACY_PDPES; i++) { 920 u64 daddr = i915_page_dir_dma_addr(ppgtt, i); 921 922 I915_WRITE(offset, lower_32_bits(daddr)); 923 I915_WRITE(offset + 4, upper_32_bits(daddr)); 924 925 offset += 8; 926 } 927 928 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : 929 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); 930 } 931 932 I915_WRITE(vgtif_reg(g2v_notify), msg); 933 934 return 0; 935 } 936 937 static void gen8_free_scratch(struct i915_address_space *vm) 938 { 939 struct drm_device *dev = vm->dev; 940 941 if (USES_FULL_48BIT_PPGTT(dev)) 942 free_pdp(dev, vm->scratch_pdp); 943 free_pd(dev, vm->scratch_pd); 944 free_pt(dev, vm->scratch_pt); 945 free_scratch_page(dev, vm->scratch_page); 946 } 947 948 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, 949 struct i915_page_directory_pointer *pdp) 950 { 951 int i; 952 953 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { 954 if (WARN_ON(!pdp->page_directory[i])) 955 continue; 956 957 gen8_free_page_tables(dev, pdp->page_directory[i]); 958 free_pd(dev, pdp->page_directory[i]); 959 } 960 961 free_pdp(dev, pdp); 962 } 963 964 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 965 { 966 int i; 967 968 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { 969 if (WARN_ON(!ppgtt->pml4.pdps[i])) 970 continue; 971 972 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); 973 } 974 975 cleanup_px(ppgtt->base.dev, &ppgtt->pml4); 976 } 977 978 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 979 { 980 struct i915_hw_ppgtt *ppgtt = 981 container_of(vm, struct i915_hw_ppgtt, base); 982 983 if (intel_vgpu_active(vm->dev)) 984 gen8_ppgtt_notify_vgt(ppgtt, false); 985 986 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 987 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); 988 else 989 gen8_ppgtt_cleanup_4lvl(ppgtt); 990 991 gen8_free_scratch(vm); 992 } 993 994 /** 995 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. 996 * @vm: Master vm structure. 997 * @pd: Page directory for this address range. 998 * @start: Starting virtual address to begin allocations. 999 * @length: Size of the allocations. 1000 * @new_pts: Bitmap set by function with new allocations. Likely used by the 1001 * caller to free on error. 1002 * 1003 * Allocate the required number of page tables. Extremely similar to 1004 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by 1005 * the page directory boundary (instead of the page directory pointer). That 1006 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is 1007 * possible, and likely that the caller will need to use multiple calls of this 1008 * function to achieve the appropriate allocation. 1009 * 1010 * Return: 0 if success; negative error code otherwise. 1011 */ 1012 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, 1013 struct i915_page_directory *pd, 1014 uint64_t start, 1015 uint64_t length, 1016 unsigned long *new_pts) 1017 { 1018 struct drm_device *dev = vm->dev; 1019 struct i915_page_table *pt; 1020 uint64_t temp; 1021 uint32_t pde; 1022 1023 gen8_for_each_pde(pt, pd, start, length, temp, pde) { 1024 /* Don't reallocate page tables */ 1025 if (test_bit(pde, pd->used_pdes)) { 1026 /* Scratch is never allocated this way */ 1027 WARN_ON(pt == vm->scratch_pt); 1028 continue; 1029 } 1030 1031 pt = alloc_pt(dev); 1032 if (IS_ERR(pt)) 1033 goto unwind_out; 1034 1035 gen8_initialize_pt(vm, pt); 1036 pd->page_table[pde] = pt; 1037 __set_bit(pde, new_pts); 1038 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); 1039 } 1040 1041 return 0; 1042 1043 unwind_out: 1044 for_each_set_bit(pde, new_pts, I915_PDES) 1045 free_pt(dev, pd->page_table[pde]); 1046 1047 return -ENOMEM; 1048 } 1049 1050 /** 1051 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. 1052 * @vm: Master vm structure. 1053 * @pdp: Page directory pointer for this address range. 1054 * @start: Starting virtual address to begin allocations. 1055 * @length: Size of the allocations. 1056 * @new_pds: Bitmap set by function with new allocations. Likely used by the 1057 * caller to free on error. 1058 * 1059 * Allocate the required number of page directories starting at the pde index of 1060 * @start, and ending at the pde index @start + @length. This function will skip 1061 * over already allocated page directories within the range, and only allocate 1062 * new ones, setting the appropriate pointer within the pdp as well as the 1063 * correct position in the bitmap @new_pds. 1064 * 1065 * The function will only allocate the pages within the range for a give page 1066 * directory pointer. In other words, if @start + @length straddles a virtually 1067 * addressed PDP boundary (512GB for 4k pages), there will be more allocations 1068 * required by the caller, This is not currently possible, and the BUG in the 1069 * code will prevent it. 1070 * 1071 * Return: 0 if success; negative error code otherwise. 1072 */ 1073 static int 1074 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, 1075 struct i915_page_directory_pointer *pdp, 1076 uint64_t start, 1077 uint64_t length, 1078 unsigned long *new_pds) 1079 { 1080 struct drm_device *dev = vm->dev; 1081 struct i915_page_directory *pd; 1082 uint64_t temp; 1083 uint32_t pdpe; 1084 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1085 1086 WARN_ON(!bitmap_empty(new_pds, pdpes)); 1087 1088 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1089 if (test_bit(pdpe, pdp->used_pdpes)) 1090 continue; 1091 1092 pd = alloc_pd(dev); 1093 if (IS_ERR(pd)) 1094 goto unwind_out; 1095 1096 gen8_initialize_pd(vm, pd); 1097 pdp->page_directory[pdpe] = pd; 1098 __set_bit(pdpe, new_pds); 1099 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); 1100 } 1101 1102 return 0; 1103 1104 unwind_out: 1105 for_each_set_bit(pdpe, new_pds, pdpes) 1106 free_pd(dev, pdp->page_directory[pdpe]); 1107 1108 return -ENOMEM; 1109 } 1110 1111 /** 1112 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. 1113 * @vm: Master vm structure. 1114 * @pml4: Page map level 4 for this address range. 1115 * @start: Starting virtual address to begin allocations. 1116 * @length: Size of the allocations. 1117 * @new_pdps: Bitmap set by function with new allocations. Likely used by the 1118 * caller to free on error. 1119 * 1120 * Allocate the required number of page directory pointers. Extremely similar to 1121 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). 1122 * The main difference is here we are limited by the pml4 boundary (instead of 1123 * the page directory pointer). 1124 * 1125 * Return: 0 if success; negative error code otherwise. 1126 */ 1127 static int 1128 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, 1129 struct i915_pml4 *pml4, 1130 uint64_t start, 1131 uint64_t length, 1132 unsigned long *new_pdps) 1133 { 1134 struct drm_device *dev = vm->dev; 1135 struct i915_page_directory_pointer *pdp; 1136 uint64_t temp; 1137 uint32_t pml4e; 1138 1139 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); 1140 1141 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1142 if (!test_bit(pml4e, pml4->used_pml4es)) { 1143 pdp = alloc_pdp(dev); 1144 if (IS_ERR(pdp)) 1145 goto unwind_out; 1146 1147 gen8_initialize_pdp(vm, pdp); 1148 pml4->pdps[pml4e] = pdp; 1149 __set_bit(pml4e, new_pdps); 1150 trace_i915_page_directory_pointer_entry_alloc(vm, 1151 pml4e, 1152 start, 1153 GEN8_PML4E_SHIFT); 1154 } 1155 } 1156 1157 return 0; 1158 1159 unwind_out: 1160 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1161 free_pdp(dev, pml4->pdps[pml4e]); 1162 1163 return -ENOMEM; 1164 } 1165 1166 static void 1167 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) 1168 { 1169 kfree(new_pts); 1170 kfree(new_pds); 1171 } 1172 1173 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both 1174 * of these are based on the number of PDPEs in the system. 1175 */ 1176 static 1177 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, 1178 unsigned long **new_pts, 1179 uint32_t pdpes) 1180 { 1181 unsigned long *pds; 1182 unsigned long *pts; 1183 1184 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); 1185 if (!pds) 1186 return -ENOMEM; 1187 1188 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), 1189 GFP_TEMPORARY); 1190 if (!pts) 1191 goto err_out; 1192 1193 *new_pds = pds; 1194 *new_pts = pts; 1195 1196 return 0; 1197 1198 err_out: 1199 free_gen8_temp_bitmaps(pds, pts); 1200 return -ENOMEM; 1201 } 1202 1203 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 1204 * the page table structures, we mark them dirty so that 1205 * context switching/execlist queuing code takes extra steps 1206 * to ensure that tlbs are flushed. 1207 */ 1208 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 1209 { 1210 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 1211 } 1212 1213 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, 1214 struct i915_page_directory_pointer *pdp, 1215 uint64_t start, 1216 uint64_t length) 1217 { 1218 struct i915_hw_ppgtt *ppgtt = 1219 container_of(vm, struct i915_hw_ppgtt, base); 1220 unsigned long *new_page_dirs, *new_page_tables; 1221 struct drm_device *dev = vm->dev; 1222 struct i915_page_directory *pd; 1223 const uint64_t orig_start = start; 1224 const uint64_t orig_length = length; 1225 uint64_t temp; 1226 uint32_t pdpe; 1227 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1228 int ret; 1229 1230 /* Wrap is never okay since we can only represent 48b, and we don't 1231 * actually use the other side of the canonical address space. 1232 */ 1233 if (WARN_ON(start + length < start)) 1234 return -ENODEV; 1235 1236 if (WARN_ON(start + length > vm->total)) 1237 return -ENODEV; 1238 1239 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1240 if (ret) 1241 return ret; 1242 1243 /* Do the allocations first so we can easily bail out */ 1244 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, 1245 new_page_dirs); 1246 if (ret) { 1247 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1248 return ret; 1249 } 1250 1251 /* For every page directory referenced, allocate page tables */ 1252 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1253 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, 1254 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); 1255 if (ret) 1256 goto err_out; 1257 } 1258 1259 start = orig_start; 1260 length = orig_length; 1261 1262 /* Allocations have completed successfully, so set the bitmaps, and do 1263 * the mappings. */ 1264 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1265 gen8_pde_t *const page_directory = kmap_px(pd); 1266 struct i915_page_table *pt; 1267 uint64_t pd_len = length; 1268 uint64_t pd_start = start; 1269 uint32_t pde; 1270 1271 /* Every pd should be allocated, we just did that above. */ 1272 WARN_ON(!pd); 1273 1274 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1275 /* Same reasoning as pd */ 1276 WARN_ON(!pt); 1277 WARN_ON(!pd_len); 1278 WARN_ON(!gen8_pte_count(pd_start, pd_len)); 1279 1280 /* Set our used ptes within the page table */ 1281 bitmap_set(pt->used_ptes, 1282 gen8_pte_index(pd_start), 1283 gen8_pte_count(pd_start, pd_len)); 1284 1285 /* Our pde is now pointing to the pagetable, pt */ 1286 __set_bit(pde, pd->used_pdes); 1287 1288 /* Map the PDE to the page table */ 1289 page_directory[pde] = gen8_pde_encode(px_dma(pt), 1290 I915_CACHE_LLC); 1291 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, 1292 gen8_pte_index(start), 1293 gen8_pte_count(start, length), 1294 GEN8_PTES); 1295 1296 /* NB: We haven't yet mapped ptes to pages. At this 1297 * point we're still relying on insert_entries() */ 1298 } 1299 1300 kunmap_px(ppgtt, page_directory); 1301 __set_bit(pdpe, pdp->used_pdpes); 1302 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); 1303 } 1304 1305 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1306 mark_tlbs_dirty(ppgtt); 1307 return 0; 1308 1309 err_out: 1310 while (pdpe--) { 1311 for_each_set_bit(temp, new_page_tables + pdpe * 1312 BITS_TO_LONGS(I915_PDES), I915_PDES) 1313 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); 1314 } 1315 1316 for_each_set_bit(pdpe, new_page_dirs, pdpes) 1317 free_pd(dev, pdp->page_directory[pdpe]); 1318 1319 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1320 mark_tlbs_dirty(ppgtt); 1321 return ret; 1322 } 1323 1324 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, 1325 struct i915_pml4 *pml4, 1326 uint64_t start, 1327 uint64_t length) 1328 { 1329 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); 1330 struct i915_hw_ppgtt *ppgtt = 1331 container_of(vm, struct i915_hw_ppgtt, base); 1332 struct i915_page_directory_pointer *pdp; 1333 uint64_t temp, pml4e; 1334 int ret = 0; 1335 1336 /* Do the pml4 allocations first, so we don't need to track the newly 1337 * allocated tables below the pdp */ 1338 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); 1339 1340 /* The pagedirectory and pagetable allocations are done in the shared 3 1341 * and 4 level code. Just allocate the pdps. 1342 */ 1343 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, 1344 new_pdps); 1345 if (ret) 1346 return ret; 1347 1348 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, 1349 "The allocation has spanned more than 512GB. " 1350 "It is highly likely this is incorrect."); 1351 1352 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1353 WARN_ON(!pdp); 1354 1355 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); 1356 if (ret) 1357 goto err_out; 1358 1359 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); 1360 } 1361 1362 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, 1363 GEN8_PML4ES_PER_PML4); 1364 1365 return 0; 1366 1367 err_out: 1368 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1369 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); 1370 1371 return ret; 1372 } 1373 1374 static int gen8_alloc_va_range(struct i915_address_space *vm, 1375 uint64_t start, uint64_t length) 1376 { 1377 struct i915_hw_ppgtt *ppgtt = 1378 container_of(vm, struct i915_hw_ppgtt, base); 1379 1380 if (USES_FULL_48BIT_PPGTT(vm->dev)) 1381 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); 1382 else 1383 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); 1384 } 1385 1386 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, 1387 uint64_t start, uint64_t length, 1388 gen8_pte_t scratch_pte, 1389 struct seq_file *m) 1390 { 1391 struct i915_page_directory *pd; 1392 uint64_t temp; 1393 uint32_t pdpe; 1394 1395 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1396 struct i915_page_table *pt; 1397 uint64_t pd_len = length; 1398 uint64_t pd_start = start; 1399 uint32_t pde; 1400 1401 if (!test_bit(pdpe, pdp->used_pdpes)) 1402 continue; 1403 1404 seq_printf(m, "\tPDPE #%d\n", pdpe); 1405 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1406 uint32_t pte; 1407 gen8_pte_t *pt_vaddr; 1408 1409 if (!test_bit(pde, pd->used_pdes)) 1410 continue; 1411 1412 pt_vaddr = kmap_px(pt); 1413 for (pte = 0; pte < GEN8_PTES; pte += 4) { 1414 uint64_t va = 1415 (pdpe << GEN8_PDPE_SHIFT) | 1416 (pde << GEN8_PDE_SHIFT) | 1417 (pte << GEN8_PTE_SHIFT); 1418 int i; 1419 bool found = false; 1420 1421 for (i = 0; i < 4; i++) 1422 if (pt_vaddr[pte + i] != scratch_pte) 1423 found = true; 1424 if (!found) 1425 continue; 1426 1427 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); 1428 for (i = 0; i < 4; i++) { 1429 if (pt_vaddr[pte + i] != scratch_pte) 1430 seq_printf(m, " %llx", pt_vaddr[pte + i]); 1431 else 1432 seq_puts(m, " SCRATCH "); 1433 } 1434 seq_puts(m, "\n"); 1435 } 1436 /* don't use kunmap_px, it could trigger 1437 * an unnecessary flush. 1438 */ 1439 kunmap_atomic(pt_vaddr); 1440 } 1441 } 1442 } 1443 1444 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1445 { 1446 struct i915_address_space *vm = &ppgtt->base; 1447 uint64_t start = ppgtt->base.start; 1448 uint64_t length = ppgtt->base.total; 1449 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 1450 I915_CACHE_LLC, true); 1451 1452 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 1453 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); 1454 } else { 1455 uint64_t templ4, pml4e; 1456 struct i915_pml4 *pml4 = &ppgtt->pml4; 1457 struct i915_page_directory_pointer *pdp; 1458 1459 gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) { 1460 if (!test_bit(pml4e, pml4->used_pml4es)) 1461 continue; 1462 1463 seq_printf(m, " PML4E #%llu\n", pml4e); 1464 gen8_dump_pdp(pdp, start, length, scratch_pte, m); 1465 } 1466 } 1467 } 1468 1469 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) 1470 { 1471 unsigned long *new_page_dirs, *new_page_tables; 1472 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1473 int ret; 1474 1475 /* We allocate temp bitmap for page tables for no gain 1476 * but as this is for init only, lets keep the things simple 1477 */ 1478 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1479 if (ret) 1480 return ret; 1481 1482 /* Allocate for all pdps regardless of how the ppgtt 1483 * was defined. 1484 */ 1485 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, 1486 0, 1ULL << 32, 1487 new_page_dirs); 1488 if (!ret) 1489 *ppgtt->pdp.used_pdpes = *new_page_dirs; 1490 1491 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1492 1493 return ret; 1494 } 1495 1496 /* 1497 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1498 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1499 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1500 * space. 1501 * 1502 */ 1503 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1504 { 1505 int ret; 1506 1507 ret = gen8_init_scratch(&ppgtt->base); 1508 if (ret) 1509 return ret; 1510 1511 ppgtt->base.start = 0; 1512 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1513 ppgtt->base.allocate_va_range = gen8_alloc_va_range; 1514 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; 1515 ppgtt->base.clear_range = gen8_ppgtt_clear_range; 1516 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1517 ppgtt->base.bind_vma = ppgtt_bind_vma; 1518 ppgtt->debug_dump = gen8_dump_ppgtt; 1519 1520 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { 1521 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); 1522 if (ret) 1523 goto free_scratch; 1524 1525 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); 1526 1527 ppgtt->base.total = 1ULL << 48; 1528 ppgtt->switch_mm = gen8_48b_mm_switch; 1529 } else { 1530 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); 1531 if (ret) 1532 goto free_scratch; 1533 1534 ppgtt->base.total = 1ULL << 32; 1535 ppgtt->switch_mm = gen8_legacy_mm_switch; 1536 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, 1537 0, 0, 1538 GEN8_PML4E_SHIFT); 1539 1540 if (intel_vgpu_active(ppgtt->base.dev)) { 1541 ret = gen8_preallocate_top_level_pdps(ppgtt); 1542 if (ret) 1543 goto free_scratch; 1544 } 1545 } 1546 1547 if (intel_vgpu_active(ppgtt->base.dev)) 1548 gen8_ppgtt_notify_vgt(ppgtt, true); 1549 1550 return 0; 1551 1552 free_scratch: 1553 gen8_free_scratch(&ppgtt->base); 1554 return ret; 1555 } 1556 1557 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1558 { 1559 struct i915_address_space *vm = &ppgtt->base; 1560 struct i915_page_table *unused; 1561 gen6_pte_t scratch_pte; 1562 uint32_t pd_entry; 1563 uint32_t pte, pde, temp; 1564 uint32_t start = ppgtt->base.start, length = ppgtt->base.total; 1565 1566 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1567 I915_CACHE_LLC, true, 0); 1568 1569 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { 1570 u32 expected; 1571 gen6_pte_t *pt_vaddr; 1572 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1573 pd_entry = readl(ppgtt->pd_addr + pde); 1574 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1575 1576 if (pd_entry != expected) 1577 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1578 pde, 1579 pd_entry, 1580 expected); 1581 seq_printf(m, "\tPDE: %x\n", pd_entry); 1582 1583 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); 1584 1585 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1586 unsigned long va = 1587 (pde * PAGE_SIZE * GEN6_PTES) + 1588 (pte * PAGE_SIZE); 1589 int i; 1590 bool found = false; 1591 for (i = 0; i < 4; i++) 1592 if (pt_vaddr[pte + i] != scratch_pte) 1593 found = true; 1594 if (!found) 1595 continue; 1596 1597 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1598 for (i = 0; i < 4; i++) { 1599 if (pt_vaddr[pte + i] != scratch_pte) 1600 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1601 else 1602 seq_puts(m, " SCRATCH "); 1603 } 1604 seq_puts(m, "\n"); 1605 } 1606 kunmap_px(ppgtt, pt_vaddr); 1607 } 1608 } 1609 1610 /* Write pde (index) from the page directory @pd to the page table @pt */ 1611 static void gen6_write_pde(struct i915_page_directory *pd, 1612 const int pde, struct i915_page_table *pt) 1613 { 1614 /* Caller needs to make sure the write completes if necessary */ 1615 struct i915_hw_ppgtt *ppgtt = 1616 container_of(pd, struct i915_hw_ppgtt, pd); 1617 u32 pd_entry; 1618 1619 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); 1620 pd_entry |= GEN6_PDE_VALID; 1621 1622 writel(pd_entry, ppgtt->pd_addr + pde); 1623 } 1624 1625 /* Write all the page tables found in the ppgtt structure to incrementing page 1626 * directories. */ 1627 static void gen6_write_page_range(struct drm_i915_private *dev_priv, 1628 struct i915_page_directory *pd, 1629 uint32_t start, uint32_t length) 1630 { 1631 struct i915_page_table *pt; 1632 uint32_t pde, temp; 1633 1634 gen6_for_each_pde(pt, pd, start, length, temp, pde) 1635 gen6_write_pde(pd, pde, pt); 1636 1637 /* Make sure write is complete before other code can use this page 1638 * table. Also require for WC mapped PTEs */ 1639 readl(dev_priv->gtt.gsm); 1640 } 1641 1642 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1643 { 1644 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1645 1646 return (ppgtt->pd.base.ggtt_offset / 64) << 16; 1647 } 1648 1649 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1650 struct drm_i915_gem_request *req) 1651 { 1652 struct intel_engine_cs *ring = req->ring; 1653 int ret; 1654 1655 /* NB: TLBs must be flushed and invalidated before a switch */ 1656 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1657 if (ret) 1658 return ret; 1659 1660 ret = intel_ring_begin(req, 6); 1661 if (ret) 1662 return ret; 1663 1664 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1665 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1666 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1667 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1668 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1669 intel_ring_emit(ring, MI_NOOP); 1670 intel_ring_advance(ring); 1671 1672 return 0; 1673 } 1674 1675 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, 1676 struct drm_i915_gem_request *req) 1677 { 1678 struct intel_engine_cs *ring = req->ring; 1679 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); 1680 1681 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1682 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1683 return 0; 1684 } 1685 1686 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1687 struct drm_i915_gem_request *req) 1688 { 1689 struct intel_engine_cs *ring = req->ring; 1690 int ret; 1691 1692 /* NB: TLBs must be flushed and invalidated before a switch */ 1693 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1694 if (ret) 1695 return ret; 1696 1697 ret = intel_ring_begin(req, 6); 1698 if (ret) 1699 return ret; 1700 1701 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1702 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1703 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1704 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1705 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1706 intel_ring_emit(ring, MI_NOOP); 1707 intel_ring_advance(ring); 1708 1709 /* XXX: RCS is the only one to auto invalidate the TLBs? */ 1710 if (ring->id != RCS) { 1711 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1712 if (ret) 1713 return ret; 1714 } 1715 1716 return 0; 1717 } 1718 1719 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1720 struct drm_i915_gem_request *req) 1721 { 1722 struct intel_engine_cs *ring = req->ring; 1723 struct drm_device *dev = ppgtt->base.dev; 1724 struct drm_i915_private *dev_priv = dev->dev_private; 1725 1726 1727 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1728 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1729 1730 POSTING_READ(RING_PP_DIR_DCLV(ring)); 1731 1732 return 0; 1733 } 1734 1735 static void gen8_ppgtt_enable(struct drm_device *dev) 1736 { 1737 struct drm_i915_private *dev_priv = dev->dev_private; 1738 struct intel_engine_cs *ring; 1739 int j; 1740 1741 for_each_ring(ring, dev_priv, j) { 1742 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; 1743 I915_WRITE(RING_MODE_GEN7(ring), 1744 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1745 } 1746 } 1747 1748 static void gen7_ppgtt_enable(struct drm_device *dev) 1749 { 1750 struct drm_i915_private *dev_priv = dev->dev_private; 1751 struct intel_engine_cs *ring; 1752 uint32_t ecochk, ecobits; 1753 int i; 1754 1755 ecobits = I915_READ(GAC_ECO_BITS); 1756 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1757 1758 ecochk = I915_READ(GAM_ECOCHK); 1759 if (IS_HASWELL(dev)) { 1760 ecochk |= ECOCHK_PPGTT_WB_HSW; 1761 } else { 1762 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1763 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1764 } 1765 I915_WRITE(GAM_ECOCHK, ecochk); 1766 1767 for_each_ring(ring, dev_priv, i) { 1768 /* GFX_MODE is per-ring on gen7+ */ 1769 I915_WRITE(RING_MODE_GEN7(ring), 1770 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1771 } 1772 } 1773 1774 static void gen6_ppgtt_enable(struct drm_device *dev) 1775 { 1776 struct drm_i915_private *dev_priv = dev->dev_private; 1777 uint32_t ecochk, gab_ctl, ecobits; 1778 1779 ecobits = I915_READ(GAC_ECO_BITS); 1780 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1781 ECOBITS_PPGTT_CACHE64B); 1782 1783 gab_ctl = I915_READ(GAB_CTL); 1784 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1785 1786 ecochk = I915_READ(GAM_ECOCHK); 1787 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1788 1789 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1790 } 1791 1792 /* PPGTT support for Sandybdrige/Gen6 and later */ 1793 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1794 uint64_t start, 1795 uint64_t length, 1796 bool use_scratch) 1797 { 1798 struct i915_hw_ppgtt *ppgtt = 1799 container_of(vm, struct i915_hw_ppgtt, base); 1800 gen6_pte_t *pt_vaddr, scratch_pte; 1801 unsigned first_entry = start >> PAGE_SHIFT; 1802 unsigned num_entries = length >> PAGE_SHIFT; 1803 unsigned act_pt = first_entry / GEN6_PTES; 1804 unsigned first_pte = first_entry % GEN6_PTES; 1805 unsigned last_pte, i; 1806 1807 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1808 I915_CACHE_LLC, true, 0); 1809 1810 while (num_entries) { 1811 last_pte = first_pte + num_entries; 1812 if (last_pte > GEN6_PTES) 1813 last_pte = GEN6_PTES; 1814 1815 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1816 1817 for (i = first_pte; i < last_pte; i++) 1818 pt_vaddr[i] = scratch_pte; 1819 1820 kunmap_px(ppgtt, pt_vaddr); 1821 1822 num_entries -= last_pte - first_pte; 1823 first_pte = 0; 1824 act_pt++; 1825 } 1826 } 1827 1828 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 1829 struct sg_table *pages, 1830 uint64_t start, 1831 enum i915_cache_level cache_level, u32 flags) 1832 { 1833 struct i915_hw_ppgtt *ppgtt = 1834 container_of(vm, struct i915_hw_ppgtt, base); 1835 gen6_pte_t *pt_vaddr; 1836 unsigned first_entry = start >> PAGE_SHIFT; 1837 unsigned act_pt = first_entry / GEN6_PTES; 1838 unsigned act_pte = first_entry % GEN6_PTES; 1839 struct sg_page_iter sg_iter; 1840 1841 pt_vaddr = NULL; 1842 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 1843 if (pt_vaddr == NULL) 1844 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1845 1846 pt_vaddr[act_pte] = 1847 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), 1848 cache_level, true, flags); 1849 1850 if (++act_pte == GEN6_PTES) { 1851 kunmap_px(ppgtt, pt_vaddr); 1852 pt_vaddr = NULL; 1853 act_pt++; 1854 act_pte = 0; 1855 } 1856 } 1857 if (pt_vaddr) 1858 kunmap_px(ppgtt, pt_vaddr); 1859 } 1860 1861 static int gen6_alloc_va_range(struct i915_address_space *vm, 1862 uint64_t start_in, uint64_t length_in) 1863 { 1864 DECLARE_BITMAP(new_page_tables, I915_PDES); 1865 struct drm_device *dev = vm->dev; 1866 struct drm_i915_private *dev_priv = dev->dev_private; 1867 struct i915_hw_ppgtt *ppgtt = 1868 container_of(vm, struct i915_hw_ppgtt, base); 1869 struct i915_page_table *pt; 1870 uint32_t start, length, start_save, length_save; 1871 uint32_t pde, temp; 1872 int ret; 1873 1874 if (WARN_ON(start_in + length_in > ppgtt->base.total)) 1875 return -ENODEV; 1876 1877 start = start_save = start_in; 1878 length = length_save = length_in; 1879 1880 bitmap_zero(new_page_tables, I915_PDES); 1881 1882 /* The allocation is done in two stages so that we can bail out with 1883 * minimal amount of pain. The first stage finds new page tables that 1884 * need allocation. The second stage marks use ptes within the page 1885 * tables. 1886 */ 1887 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1888 if (pt != vm->scratch_pt) { 1889 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); 1890 continue; 1891 } 1892 1893 /* We've already allocated a page table */ 1894 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 1895 1896 pt = alloc_pt(dev); 1897 if (IS_ERR(pt)) { 1898 ret = PTR_ERR(pt); 1899 goto unwind_out; 1900 } 1901 1902 gen6_initialize_pt(vm, pt); 1903 1904 ppgtt->pd.page_table[pde] = pt; 1905 __set_bit(pde, new_page_tables); 1906 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); 1907 } 1908 1909 start = start_save; 1910 length = length_save; 1911 1912 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1913 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); 1914 1915 bitmap_zero(tmp_bitmap, GEN6_PTES); 1916 bitmap_set(tmp_bitmap, gen6_pte_index(start), 1917 gen6_pte_count(start, length)); 1918 1919 if (__test_and_clear_bit(pde, new_page_tables)) 1920 gen6_write_pde(&ppgtt->pd, pde, pt); 1921 1922 trace_i915_page_table_entry_map(vm, pde, pt, 1923 gen6_pte_index(start), 1924 gen6_pte_count(start, length), 1925 GEN6_PTES); 1926 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, 1927 GEN6_PTES); 1928 } 1929 1930 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); 1931 1932 /* Make sure write is complete before other code can use this page 1933 * table. Also require for WC mapped PTEs */ 1934 readl(dev_priv->gtt.gsm); 1935 1936 mark_tlbs_dirty(ppgtt); 1937 return 0; 1938 1939 unwind_out: 1940 for_each_set_bit(pde, new_page_tables, I915_PDES) { 1941 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 1942 1943 ppgtt->pd.page_table[pde] = vm->scratch_pt; 1944 free_pt(vm->dev, pt); 1945 } 1946 1947 mark_tlbs_dirty(ppgtt); 1948 return ret; 1949 } 1950 1951 static int gen6_init_scratch(struct i915_address_space *vm) 1952 { 1953 struct drm_device *dev = vm->dev; 1954 1955 vm->scratch_page = alloc_scratch_page(dev); 1956 if (IS_ERR(vm->scratch_page)) 1957 return PTR_ERR(vm->scratch_page); 1958 1959 vm->scratch_pt = alloc_pt(dev); 1960 if (IS_ERR(vm->scratch_pt)) { 1961 free_scratch_page(dev, vm->scratch_page); 1962 return PTR_ERR(vm->scratch_pt); 1963 } 1964 1965 gen6_initialize_pt(vm, vm->scratch_pt); 1966 1967 return 0; 1968 } 1969 1970 static void gen6_free_scratch(struct i915_address_space *vm) 1971 { 1972 struct drm_device *dev = vm->dev; 1973 1974 free_pt(dev, vm->scratch_pt); 1975 free_scratch_page(dev, vm->scratch_page); 1976 } 1977 1978 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 1979 { 1980 struct i915_hw_ppgtt *ppgtt = 1981 container_of(vm, struct i915_hw_ppgtt, base); 1982 struct i915_page_table *pt; 1983 uint32_t pde; 1984 1985 drm_mm_remove_node(&ppgtt->node); 1986 1987 gen6_for_all_pdes(pt, ppgtt, pde) { 1988 if (pt != vm->scratch_pt) 1989 free_pt(ppgtt->base.dev, pt); 1990 } 1991 1992 gen6_free_scratch(vm); 1993 } 1994 1995 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 1996 { 1997 struct i915_address_space *vm = &ppgtt->base; 1998 struct drm_device *dev = ppgtt->base.dev; 1999 struct drm_i915_private *dev_priv = dev->dev_private; 2000 bool retried = false; 2001 int ret; 2002 2003 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 2004 * allocator works in address space sizes, so it's multiplied by page 2005 * size. We allocate at the top of the GTT to avoid fragmentation. 2006 */ 2007 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); 2008 2009 ret = gen6_init_scratch(vm); 2010 if (ret) 2011 return ret; 2012 2013 alloc: 2014 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, 2015 &ppgtt->node, GEN6_PD_SIZE, 2016 GEN6_PD_ALIGN, 0, 2017 0, dev_priv->gtt.base.total, 2018 DRM_MM_TOPDOWN); 2019 if (ret == -ENOSPC && !retried) { 2020 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, 2021 GEN6_PD_SIZE, GEN6_PD_ALIGN, 2022 I915_CACHE_NONE, 2023 0, dev_priv->gtt.base.total, 2024 0); 2025 if (ret) 2026 goto err_out; 2027 2028 retried = true; 2029 goto alloc; 2030 } 2031 2032 if (ret) 2033 goto err_out; 2034 2035 2036 if (ppgtt->node.start < dev_priv->gtt.mappable_end) 2037 DRM_DEBUG("Forced to use aperture for PDEs\n"); 2038 2039 return 0; 2040 2041 err_out: 2042 gen6_free_scratch(vm); 2043 return ret; 2044 } 2045 2046 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 2047 { 2048 return gen6_ppgtt_allocate_page_directories(ppgtt); 2049 } 2050 2051 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 2052 uint64_t start, uint64_t length) 2053 { 2054 struct i915_page_table *unused; 2055 uint32_t pde, temp; 2056 2057 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) 2058 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 2059 } 2060 2061 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 2062 { 2063 struct drm_device *dev = ppgtt->base.dev; 2064 struct drm_i915_private *dev_priv = dev->dev_private; 2065 int ret; 2066 2067 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; 2068 if (IS_GEN6(dev)) { 2069 ppgtt->switch_mm = gen6_mm_switch; 2070 } else if (IS_HASWELL(dev)) { 2071 ppgtt->switch_mm = hsw_mm_switch; 2072 } else if (IS_GEN7(dev)) { 2073 ppgtt->switch_mm = gen7_mm_switch; 2074 } else 2075 BUG(); 2076 2077 if (intel_vgpu_active(dev)) 2078 ppgtt->switch_mm = vgpu_mm_switch; 2079 2080 ret = gen6_ppgtt_alloc(ppgtt); 2081 if (ret) 2082 return ret; 2083 2084 ppgtt->base.allocate_va_range = gen6_alloc_va_range; 2085 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 2086 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 2087 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 2088 ppgtt->base.bind_vma = ppgtt_bind_vma; 2089 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 2090 ppgtt->base.start = 0; 2091 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 2092 ppgtt->debug_dump = gen6_dump_ppgtt; 2093 2094 ppgtt->pd.base.ggtt_offset = 2095 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 2096 2097 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + 2098 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 2099 2100 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 2101 2102 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); 2103 2104 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", 2105 ppgtt->node.size >> 20, 2106 ppgtt->node.start / PAGE_SIZE); 2107 2108 DRM_DEBUG("Adding PPGTT at offset %x\n", 2109 ppgtt->pd.base.ggtt_offset << 10); 2110 2111 return 0; 2112 } 2113 2114 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2115 { 2116 ppgtt->base.dev = dev; 2117 2118 if (INTEL_INFO(dev)->gen < 8) 2119 return gen6_ppgtt_init(ppgtt); 2120 else 2121 return gen8_ppgtt_init(ppgtt); 2122 } 2123 2124 static void i915_address_space_init(struct i915_address_space *vm, 2125 struct drm_i915_private *dev_priv) 2126 { 2127 drm_mm_init(&vm->mm, vm->start, vm->total); 2128 vm->dev = dev_priv->dev; 2129 INIT_LIST_HEAD(&vm->active_list); 2130 INIT_LIST_HEAD(&vm->inactive_list); 2131 list_add_tail(&vm->global_link, &dev_priv->vm_list); 2132 } 2133 2134 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2135 { 2136 struct drm_i915_private *dev_priv = dev->dev_private; 2137 int ret = 0; 2138 2139 ret = __hw_ppgtt_init(dev, ppgtt); 2140 if (ret == 0) { 2141 kref_init(&ppgtt->ref); 2142 i915_address_space_init(&ppgtt->base, dev_priv); 2143 } 2144 2145 return ret; 2146 } 2147 2148 int i915_ppgtt_init_hw(struct drm_device *dev) 2149 { 2150 /* In the case of execlists, PPGTT is enabled by the context descriptor 2151 * and the PDPs are contained within the context itself. We don't 2152 * need to do anything here. */ 2153 if (i915.enable_execlists) 2154 return 0; 2155 2156 if (!USES_PPGTT(dev)) 2157 return 0; 2158 2159 if (IS_GEN6(dev)) 2160 gen6_ppgtt_enable(dev); 2161 else if (IS_GEN7(dev)) 2162 gen7_ppgtt_enable(dev); 2163 else if (INTEL_INFO(dev)->gen >= 8) 2164 gen8_ppgtt_enable(dev); 2165 else 2166 MISSING_CASE(INTEL_INFO(dev)->gen); 2167 2168 return 0; 2169 } 2170 2171 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) 2172 { 2173 struct drm_i915_private *dev_priv = req->ring->dev->dev_private; 2174 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2175 2176 if (i915.enable_execlists) 2177 return 0; 2178 2179 if (!ppgtt) 2180 return 0; 2181 2182 return ppgtt->switch_mm(ppgtt, req); 2183 } 2184 2185 struct i915_hw_ppgtt * 2186 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) 2187 { 2188 struct i915_hw_ppgtt *ppgtt; 2189 int ret; 2190 2191 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2192 if (!ppgtt) 2193 return ERR_PTR(-ENOMEM); 2194 2195 ret = i915_ppgtt_init(dev, ppgtt); 2196 if (ret) { 2197 kfree(ppgtt); 2198 return ERR_PTR(ret); 2199 } 2200 2201 ppgtt->file_priv = fpriv; 2202 2203 trace_i915_ppgtt_create(&ppgtt->base); 2204 2205 return ppgtt; 2206 } 2207 2208 void i915_ppgtt_release(struct kref *kref) 2209 { 2210 struct i915_hw_ppgtt *ppgtt = 2211 container_of(kref, struct i915_hw_ppgtt, ref); 2212 2213 trace_i915_ppgtt_release(&ppgtt->base); 2214 2215 /* vmas should already be unbound */ 2216 WARN_ON(!list_empty(&ppgtt->base.active_list)); 2217 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 2218 2219 list_del(&ppgtt->base.global_link); 2220 drm_mm_takedown(&ppgtt->base.mm); 2221 2222 ppgtt->base.cleanup(&ppgtt->base); 2223 kfree(ppgtt); 2224 } 2225 2226 extern int intel_iommu_gfx_mapped; 2227 /* Certain Gen5 chipsets require require idling the GPU before 2228 * unmapping anything from the GTT when VT-d is enabled. 2229 */ 2230 static bool needs_idle_maps(struct drm_device *dev) 2231 { 2232 #ifdef CONFIG_INTEL_IOMMU 2233 /* Query intel_iommu to see if we need the workaround. Presumably that 2234 * was loaded first. 2235 */ 2236 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 2237 return true; 2238 #endif 2239 return false; 2240 } 2241 2242 static bool do_idling(struct drm_i915_private *dev_priv) 2243 { 2244 bool ret = dev_priv->mm.interruptible; 2245 2246 if (unlikely(dev_priv->gtt.do_idle_maps)) { 2247 dev_priv->mm.interruptible = false; 2248 if (i915_gpu_idle(dev_priv->dev)) { 2249 DRM_ERROR("Couldn't idle GPU\n"); 2250 /* Wait a bit, in hopes it avoids the hang */ 2251 udelay(10); 2252 } 2253 } 2254 2255 return ret; 2256 } 2257 2258 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 2259 { 2260 if (unlikely(dev_priv->gtt.do_idle_maps)) 2261 dev_priv->mm.interruptible = interruptible; 2262 } 2263 2264 void i915_check_and_clear_faults(struct drm_device *dev) 2265 { 2266 struct drm_i915_private *dev_priv = dev->dev_private; 2267 struct intel_engine_cs *ring; 2268 int i; 2269 2270 if (INTEL_INFO(dev)->gen < 6) 2271 return; 2272 2273 for_each_ring(ring, dev_priv, i) { 2274 u32 fault_reg; 2275 fault_reg = I915_READ(RING_FAULT_REG(ring)); 2276 if (fault_reg & RING_FAULT_VALID) { 2277 DRM_DEBUG_DRIVER("Unexpected fault\n" 2278 "\tAddr: 0x%08lx\n" 2279 "\tAddress space: %s\n" 2280 "\tSource ID: %d\n" 2281 "\tType: %d\n", 2282 fault_reg & PAGE_MASK, 2283 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 2284 RING_FAULT_SRCID(fault_reg), 2285 RING_FAULT_FAULT_TYPE(fault_reg)); 2286 I915_WRITE(RING_FAULT_REG(ring), 2287 fault_reg & ~RING_FAULT_VALID); 2288 } 2289 } 2290 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); 2291 } 2292 2293 static void i915_ggtt_flush(struct drm_i915_private *dev_priv) 2294 { 2295 if (INTEL_INFO(dev_priv->dev)->gen < 6) { 2296 intel_gtt_chipset_flush(); 2297 } else { 2298 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2299 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2300 } 2301 } 2302 2303 void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 2304 { 2305 struct drm_i915_private *dev_priv = dev->dev_private; 2306 2307 /* Don't bother messing with faults pre GEN6 as we have little 2308 * documentation supporting that it's a good idea. 2309 */ 2310 if (INTEL_INFO(dev)->gen < 6) 2311 return; 2312 2313 i915_check_and_clear_faults(dev); 2314 2315 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 2316 dev_priv->gtt.base.start, 2317 dev_priv->gtt.base.total, 2318 true); 2319 2320 i915_ggtt_flush(dev_priv); 2321 } 2322 2323 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 2324 { 2325 if (!dma_map_sg(&obj->base.dev->pdev->dev, 2326 obj->pages->sgl, obj->pages->nents, 2327 PCI_DMA_BIDIRECTIONAL)) 2328 return -ENOSPC; 2329 2330 return 0; 2331 } 2332 2333 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2334 { 2335 #ifdef writeq 2336 writeq(pte, addr); 2337 #else 2338 iowrite32((u32)pte, addr); 2339 iowrite32(pte >> 32, addr + 4); 2340 #endif 2341 } 2342 2343 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2344 struct sg_table *st, 2345 uint64_t start, 2346 enum i915_cache_level level, u32 unused) 2347 { 2348 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2349 unsigned first_entry = start >> PAGE_SHIFT; 2350 gen8_pte_t __iomem *gtt_entries = 2351 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2352 int i = 0; 2353 struct sg_page_iter sg_iter; 2354 dma_addr_t addr = 0; /* shut up gcc */ 2355 2356 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2357 addr = sg_dma_address(sg_iter.sg) + 2358 (sg_iter.sg_pgoffset << PAGE_SHIFT); 2359 gen8_set_pte(>t_entries[i], 2360 gen8_pte_encode(addr, level, true)); 2361 i++; 2362 } 2363 2364 /* 2365 * XXX: This serves as a posting read to make sure that the PTE has 2366 * actually been updated. There is some concern that even though 2367 * registers and PTEs are within the same BAR that they are potentially 2368 * of NUMA access patterns. Therefore, even with the way we assume 2369 * hardware should work, we must keep this posting read for paranoia. 2370 */ 2371 if (i != 0) 2372 WARN_ON(readq(>t_entries[i-1]) 2373 != gen8_pte_encode(addr, level, true)); 2374 2375 /* This next bit makes the above posting read even more important. We 2376 * want to flush the TLBs only after we're certain all the PTE updates 2377 * have finished. 2378 */ 2379 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2380 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2381 } 2382 2383 /* 2384 * Binds an object into the global gtt with the specified cache level. The object 2385 * will be accessible to the GPU via commands whose operands reference offsets 2386 * within the global GTT as well as accessible by the GPU through the GMADR 2387 * mapped BAR (dev_priv->mm.gtt->gtt). 2388 */ 2389 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 2390 struct sg_table *st, 2391 uint64_t start, 2392 enum i915_cache_level level, u32 flags) 2393 { 2394 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2395 unsigned first_entry = start >> PAGE_SHIFT; 2396 gen6_pte_t __iomem *gtt_entries = 2397 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2398 int i = 0; 2399 struct sg_page_iter sg_iter; 2400 dma_addr_t addr = 0; 2401 2402 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2403 addr = sg_page_iter_dma_address(&sg_iter); 2404 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); 2405 i++; 2406 } 2407 2408 /* XXX: This serves as a posting read to make sure that the PTE has 2409 * actually been updated. There is some concern that even though 2410 * registers and PTEs are within the same BAR that they are potentially 2411 * of NUMA access patterns. Therefore, even with the way we assume 2412 * hardware should work, we must keep this posting read for paranoia. 2413 */ 2414 if (i != 0) { 2415 unsigned long gtt = readl(>t_entries[i-1]); 2416 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); 2417 } 2418 2419 /* This next bit makes the above posting read even more important. We 2420 * want to flush the TLBs only after we're certain all the PTE updates 2421 * have finished. 2422 */ 2423 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2424 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2425 } 2426 2427 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2428 uint64_t start, 2429 uint64_t length, 2430 bool use_scratch) 2431 { 2432 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2433 unsigned first_entry = start >> PAGE_SHIFT; 2434 unsigned num_entries = length >> PAGE_SHIFT; 2435 gen8_pte_t scratch_pte, __iomem *gtt_base = 2436 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2437 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2438 int i; 2439 2440 if (WARN(num_entries > max_entries, 2441 "First entry = %d; Num entries = %d (max=%d)\n", 2442 first_entry, num_entries, max_entries)) 2443 num_entries = max_entries; 2444 2445 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 2446 I915_CACHE_LLC, 2447 use_scratch); 2448 for (i = 0; i < num_entries; i++) 2449 gen8_set_pte(>t_base[i], scratch_pte); 2450 readl(gtt_base); 2451 } 2452 2453 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2454 uint64_t start, 2455 uint64_t length, 2456 bool use_scratch) 2457 { 2458 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2459 unsigned first_entry = start >> PAGE_SHIFT; 2460 unsigned num_entries = length >> PAGE_SHIFT; 2461 gen6_pte_t scratch_pte, __iomem *gtt_base = 2462 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2463 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2464 int i; 2465 2466 if (WARN(num_entries > max_entries, 2467 "First entry = %d; Num entries = %d (max=%d)\n", 2468 first_entry, num_entries, max_entries)) 2469 num_entries = max_entries; 2470 2471 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 2472 I915_CACHE_LLC, use_scratch, 0); 2473 2474 for (i = 0; i < num_entries; i++) 2475 iowrite32(scratch_pte, >t_base[i]); 2476 readl(gtt_base); 2477 } 2478 2479 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2480 struct sg_table *pages, 2481 uint64_t start, 2482 enum i915_cache_level cache_level, u32 unused) 2483 { 2484 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2485 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2486 2487 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2488 2489 } 2490 2491 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2492 uint64_t start, 2493 uint64_t length, 2494 bool unused) 2495 { 2496 unsigned first_entry = start >> PAGE_SHIFT; 2497 unsigned num_entries = length >> PAGE_SHIFT; 2498 intel_gtt_clear_range(first_entry, num_entries); 2499 } 2500 2501 static int ggtt_bind_vma(struct i915_vma *vma, 2502 enum i915_cache_level cache_level, 2503 u32 flags) 2504 { 2505 struct drm_i915_gem_object *obj = vma->obj; 2506 u32 pte_flags = 0; 2507 int ret; 2508 2509 ret = i915_get_ggtt_vma_pages(vma); 2510 if (ret) 2511 return ret; 2512 2513 /* Currently applicable only to VLV */ 2514 if (obj->gt_ro) 2515 pte_flags |= PTE_READ_ONLY; 2516 2517 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, 2518 vma->node.start, 2519 cache_level, pte_flags); 2520 2521 /* 2522 * Without aliasing PPGTT there's no difference between 2523 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 2524 * upgrade to both bound if we bind either to avoid double-binding. 2525 */ 2526 vma->bound |= GLOBAL_BIND | LOCAL_BIND; 2527 2528 return 0; 2529 } 2530 2531 static int aliasing_gtt_bind_vma(struct i915_vma *vma, 2532 enum i915_cache_level cache_level, 2533 u32 flags) 2534 { 2535 struct drm_device *dev = vma->vm->dev; 2536 struct drm_i915_private *dev_priv = dev->dev_private; 2537 struct drm_i915_gem_object *obj = vma->obj; 2538 struct sg_table *pages = obj->pages; 2539 u32 pte_flags = 0; 2540 int ret; 2541 2542 ret = i915_get_ggtt_vma_pages(vma); 2543 if (ret) 2544 return ret; 2545 pages = vma->ggtt_view.pages; 2546 2547 /* Currently applicable only to VLV */ 2548 if (obj->gt_ro) 2549 pte_flags |= PTE_READ_ONLY; 2550 2551 2552 if (flags & GLOBAL_BIND) { 2553 vma->vm->insert_entries(vma->vm, pages, 2554 vma->node.start, 2555 cache_level, pte_flags); 2556 } 2557 2558 if (flags & LOCAL_BIND) { 2559 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2560 appgtt->base.insert_entries(&appgtt->base, pages, 2561 vma->node.start, 2562 cache_level, pte_flags); 2563 } 2564 2565 return 0; 2566 } 2567 2568 static void ggtt_unbind_vma(struct i915_vma *vma) 2569 { 2570 struct drm_device *dev = vma->vm->dev; 2571 struct drm_i915_private *dev_priv = dev->dev_private; 2572 struct drm_i915_gem_object *obj = vma->obj; 2573 const uint64_t size = min_t(uint64_t, 2574 obj->base.size, 2575 vma->node.size); 2576 2577 if (vma->bound & GLOBAL_BIND) { 2578 vma->vm->clear_range(vma->vm, 2579 vma->node.start, 2580 size, 2581 true); 2582 } 2583 2584 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { 2585 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2586 2587 appgtt->base.clear_range(&appgtt->base, 2588 vma->node.start, 2589 size, 2590 true); 2591 } 2592 } 2593 2594 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 2595 { 2596 struct drm_device *dev = obj->base.dev; 2597 struct drm_i915_private *dev_priv = dev->dev_private; 2598 bool interruptible; 2599 2600 interruptible = do_idling(dev_priv); 2601 2602 dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents, 2603 PCI_DMA_BIDIRECTIONAL); 2604 2605 undo_idling(dev_priv, interruptible); 2606 } 2607 2608 static void i915_gtt_color_adjust(struct drm_mm_node *node, 2609 unsigned long color, 2610 u64 *start, 2611 u64 *end) 2612 { 2613 if (node->color != color) 2614 *start += 4096; 2615 2616 if (!list_empty(&node->node_list)) { 2617 node = list_entry(node->node_list.next, 2618 struct drm_mm_node, 2619 node_list); 2620 if (node->allocated && node->color != color) 2621 *end -= 4096; 2622 } 2623 } 2624 2625 static int i915_gem_setup_global_gtt(struct drm_device *dev, 2626 u64 start, 2627 u64 mappable_end, 2628 u64 end) 2629 { 2630 /* Let GEM Manage all of the aperture. 2631 * 2632 * However, leave one page at the end still bound to the scratch page. 2633 * There are a number of places where the hardware apparently prefetches 2634 * past the end of the object, and we've seen multiple hangs with the 2635 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2636 * aperture. One page should be enough to keep any prefetching inside 2637 * of the aperture. 2638 */ 2639 struct drm_i915_private *dev_priv = dev->dev_private; 2640 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2641 struct drm_mm_node *entry; 2642 struct drm_i915_gem_object *obj; 2643 unsigned long hole_start, hole_end; 2644 int ret; 2645 2646 BUG_ON(mappable_end > end); 2647 2648 ggtt_vm->start = start; 2649 2650 /* Subtract the guard page before address space initialization to 2651 * shrink the range used by drm_mm */ 2652 ggtt_vm->total = end - start - PAGE_SIZE; 2653 i915_address_space_init(ggtt_vm, dev_priv); 2654 ggtt_vm->total += PAGE_SIZE; 2655 2656 if (intel_vgpu_active(dev)) { 2657 ret = intel_vgt_balloon(dev); 2658 if (ret) 2659 return ret; 2660 } 2661 2662 if (!HAS_LLC(dev)) 2663 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; 2664 2665 /* Mark any preallocated objects as occupied */ 2666 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 2667 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 2668 2669 DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n", 2670 i915_gem_obj_ggtt_offset(obj), obj->base.size); 2671 2672 WARN_ON(i915_gem_obj_ggtt_bound(obj)); 2673 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); 2674 if (ret) { 2675 DRM_DEBUG_KMS("Reservation failed: %i\n", ret); 2676 return ret; 2677 } 2678 vma->bound |= GLOBAL_BIND; 2679 __i915_vma_set_map_and_fenceable(vma); 2680 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); 2681 } 2682 2683 /* Clear any non-preallocated blocks */ 2684 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { 2685 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 2686 hole_start, hole_end); 2687 ggtt_vm->clear_range(ggtt_vm, hole_start, 2688 hole_end - hole_start, true); 2689 } 2690 2691 /* And finally clear the reserved guard page */ 2692 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); 2693 2694 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { 2695 struct i915_hw_ppgtt *ppgtt; 2696 2697 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2698 if (!ppgtt) 2699 return -ENOMEM; 2700 2701 ret = __hw_ppgtt_init(dev, ppgtt); 2702 if (ret) { 2703 ppgtt->base.cleanup(&ppgtt->base); 2704 kfree(ppgtt); 2705 return ret; 2706 } 2707 2708 if (ppgtt->base.allocate_va_range) 2709 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, 2710 ppgtt->base.total); 2711 if (ret) { 2712 ppgtt->base.cleanup(&ppgtt->base); 2713 kfree(ppgtt); 2714 return ret; 2715 } 2716 2717 ppgtt->base.clear_range(&ppgtt->base, 2718 ppgtt->base.start, 2719 ppgtt->base.total, 2720 true); 2721 2722 dev_priv->mm.aliasing_ppgtt = ppgtt; 2723 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); 2724 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; 2725 } 2726 2727 return 0; 2728 } 2729 2730 void i915_gem_init_global_gtt(struct drm_device *dev) 2731 { 2732 struct drm_i915_private *dev_priv = dev->dev_private; 2733 u64 gtt_size, mappable_size; 2734 2735 gtt_size = dev_priv->gtt.base.total; 2736 mappable_size = dev_priv->gtt.mappable_end; 2737 2738 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 2739 } 2740 2741 void i915_global_gtt_cleanup(struct drm_device *dev) 2742 { 2743 struct drm_i915_private *dev_priv = dev->dev_private; 2744 struct i915_address_space *vm = &dev_priv->gtt.base; 2745 2746 if (dev_priv->mm.aliasing_ppgtt) { 2747 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2748 2749 ppgtt->base.cleanup(&ppgtt->base); 2750 } 2751 2752 if (drm_mm_initialized(&vm->mm)) { 2753 if (intel_vgpu_active(dev)) 2754 intel_vgt_deballoon(); 2755 2756 drm_mm_takedown(&vm->mm); 2757 list_del(&vm->global_link); 2758 } 2759 2760 vm->cleanup(vm); 2761 } 2762 2763 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 2764 { 2765 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 2766 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 2767 return snb_gmch_ctl << 20; 2768 } 2769 2770 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 2771 { 2772 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 2773 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 2774 if (bdw_gmch_ctl) 2775 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 2776 2777 #ifdef CONFIG_X86_32 2778 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 2779 if (bdw_gmch_ctl > 4) 2780 bdw_gmch_ctl = 4; 2781 #endif 2782 2783 return bdw_gmch_ctl << 20; 2784 } 2785 2786 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 2787 { 2788 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 2789 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 2790 2791 if (gmch_ctrl) 2792 return 1 << (20 + gmch_ctrl); 2793 2794 return 0; 2795 } 2796 2797 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 2798 { 2799 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 2800 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 2801 return snb_gmch_ctl << 25; /* 32 MB units */ 2802 } 2803 2804 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 2805 { 2806 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2807 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 2808 return bdw_gmch_ctl << 25; /* 32 MB units */ 2809 } 2810 2811 static size_t chv_get_stolen_size(u16 gmch_ctrl) 2812 { 2813 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 2814 gmch_ctrl &= SNB_GMCH_GMS_MASK; 2815 2816 /* 2817 * 0x0 to 0x10: 32MB increments starting at 0MB 2818 * 0x11 to 0x16: 4MB increments starting at 8MB 2819 * 0x17 to 0x1d: 4MB increments start at 36MB 2820 */ 2821 if (gmch_ctrl < 0x11) 2822 return gmch_ctrl << 25; 2823 else if (gmch_ctrl < 0x17) 2824 return (gmch_ctrl - 0x11 + 2) << 22; 2825 else 2826 return (gmch_ctrl - 0x17 + 9) << 22; 2827 } 2828 2829 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 2830 { 2831 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2832 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 2833 2834 if (gen9_gmch_ctl < 0xf0) 2835 return gen9_gmch_ctl << 25; /* 32 MB units */ 2836 else 2837 /* 4MB increments starting at 0xf0 for 4MB */ 2838 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 2839 } 2840 2841 static int ggtt_probe_common(struct drm_device *dev, 2842 size_t gtt_size) 2843 { 2844 struct drm_i915_private *dev_priv = dev->dev_private; 2845 struct i915_page_scratch *scratch_page; 2846 phys_addr_t gtt_phys_addr; 2847 2848 /* For Modern GENs the PTEs and register space are split in the BAR */ 2849 gtt_phys_addr = pci_resource_start(dev->pdev, 0) + 2850 (pci_resource_len(dev->pdev, 0) / 2); 2851 2852 /* 2853 * On BXT writes larger than 64 bit to the GTT pagetable range will be 2854 * dropped. For WC mappings in general we have 64 byte burst writes 2855 * when the WC buffer is flushed, so we can't use it, but have to 2856 * resort to an uncached mapping. The WC issue is easily caught by the 2857 * readback check when writing GTT PTE entries. 2858 */ 2859 if (IS_BROXTON(dev)) 2860 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); 2861 else 2862 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); 2863 if (!dev_priv->gtt.gsm) { 2864 DRM_ERROR("Failed to map the gtt page table\n"); 2865 return -ENOMEM; 2866 } 2867 2868 scratch_page = alloc_scratch_page(dev); 2869 if (IS_ERR(scratch_page)) { 2870 DRM_ERROR("Scratch setup failed\n"); 2871 /* iounmap will also get called at remove, but meh */ 2872 iounmap(dev_priv->gtt.gsm); 2873 return PTR_ERR(scratch_page); 2874 } 2875 2876 dev_priv->gtt.base.scratch_page = scratch_page; 2877 2878 return 0; 2879 } 2880 2881 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 2882 * bits. When using advanced contexts each context stores its own PAT, but 2883 * writing this data shouldn't be harmful even in those cases. */ 2884 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 2885 { 2886 uint64_t pat; 2887 2888 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 2889 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 2890 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 2891 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 2892 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 2893 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 2894 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 2895 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 2896 2897 if (!USES_PPGTT(dev_priv->dev)) 2898 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 2899 * so RTL will always use the value corresponding to 2900 * pat_sel = 000". 2901 * So let's disable cache for GGTT to avoid screen corruptions. 2902 * MOCS still can be used though. 2903 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 2904 * before this patch, i.e. the same uncached + snooping access 2905 * like on gen6/7 seems to be in effect. 2906 * - So this just fixes blitter/render access. Again it looks 2907 * like it's not just uncached access, but uncached + snooping. 2908 * So we can still hold onto all our assumptions wrt cpu 2909 * clflushing on LLC machines. 2910 */ 2911 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 2912 2913 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 2914 * write would work. */ 2915 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2916 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2917 } 2918 2919 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 2920 { 2921 uint64_t pat; 2922 2923 /* 2924 * Map WB on BDW to snooped on CHV. 2925 * 2926 * Only the snoop bit has meaning for CHV, the rest is 2927 * ignored. 2928 * 2929 * The hardware will never snoop for certain types of accesses: 2930 * - CPU GTT (GMADR->GGTT->no snoop->memory) 2931 * - PPGTT page tables 2932 * - some other special cycles 2933 * 2934 * As with BDW, we also need to consider the following for GT accesses: 2935 * "For GGTT, there is NO pat_sel[2:0] from the entry, 2936 * so RTL will always use the value corresponding to 2937 * pat_sel = 000". 2938 * Which means we must set the snoop bit in PAT entry 0 2939 * in order to keep the global status page working. 2940 */ 2941 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 2942 GEN8_PPAT(1, 0) | 2943 GEN8_PPAT(2, 0) | 2944 GEN8_PPAT(3, 0) | 2945 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 2946 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 2947 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 2948 GEN8_PPAT(7, CHV_PPAT_SNOOP); 2949 2950 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2951 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2952 } 2953 2954 static int gen8_gmch_probe(struct drm_device *dev, 2955 u64 *gtt_total, 2956 size_t *stolen, 2957 phys_addr_t *mappable_base, 2958 u64 *mappable_end) 2959 { 2960 struct drm_i915_private *dev_priv = dev->dev_private; 2961 u64 gtt_size; 2962 u16 snb_gmch_ctl; 2963 int ret; 2964 2965 /* TODO: We're not aware of mappable constraints on gen8 yet */ 2966 *mappable_base = pci_resource_start(dev->pdev, 2); 2967 *mappable_end = pci_resource_len(dev->pdev, 2); 2968 2969 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) 2970 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); 2971 2972 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2973 2974 if (INTEL_INFO(dev)->gen >= 9) { 2975 *stolen = gen9_get_stolen_size(snb_gmch_ctl); 2976 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2977 } else if (IS_CHERRYVIEW(dev)) { 2978 *stolen = chv_get_stolen_size(snb_gmch_ctl); 2979 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); 2980 } else { 2981 *stolen = gen8_get_stolen_size(snb_gmch_ctl); 2982 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2983 } 2984 2985 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 2986 2987 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 2988 chv_setup_private_ppat(dev_priv); 2989 else 2990 bdw_setup_private_ppat(dev_priv); 2991 2992 ret = ggtt_probe_common(dev, gtt_size); 2993 2994 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; 2995 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; 2996 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 2997 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 2998 2999 return ret; 3000 } 3001 3002 static int gen6_gmch_probe(struct drm_device *dev, 3003 u64 *gtt_total, 3004 size_t *stolen, 3005 phys_addr_t *mappable_base, 3006 u64 *mappable_end) 3007 { 3008 struct drm_i915_private *dev_priv = dev->dev_private; 3009 unsigned int gtt_size; 3010 u16 snb_gmch_ctl; 3011 int ret; 3012 3013 *mappable_base = pci_resource_start(dev->pdev, 2); 3014 *mappable_end = pci_resource_len(dev->pdev, 2); 3015 3016 /* 64/512MB is the current min/max we actually know of, but this is just 3017 * a coarse sanity check. 3018 */ 3019 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 3020 DRM_ERROR("Unknown GMADR size (%llx)\n", 3021 dev_priv->gtt.mappable_end); 3022 return -ENXIO; 3023 } 3024 3025 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 3026 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 3027 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3028 3029 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 3030 3031 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 3032 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 3033 3034 ret = ggtt_probe_common(dev, gtt_size); 3035 3036 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; 3037 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; 3038 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3039 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3040 3041 return ret; 3042 } 3043 3044 static void gen6_gmch_remove(struct i915_address_space *vm) 3045 { 3046 3047 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); 3048 3049 iounmap(gtt->gsm); 3050 free_scratch_page(vm->dev, vm->scratch_page); 3051 } 3052 3053 static int i915_gmch_probe(struct drm_device *dev, 3054 u64 *gtt_total, 3055 size_t *stolen, 3056 phys_addr_t *mappable_base, 3057 u64 *mappable_end) 3058 { 3059 struct drm_i915_private *dev_priv = dev->dev_private; 3060 int ret; 3061 3062 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); 3063 if (!ret) { 3064 DRM_ERROR("failed to set up gmch\n"); 3065 return -EIO; 3066 } 3067 3068 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); 3069 3070 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); 3071 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; 3072 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; 3073 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3074 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3075 3076 if (unlikely(dev_priv->gtt.do_idle_maps)) 3077 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 3078 3079 return 0; 3080 } 3081 3082 static void i915_gmch_remove(struct i915_address_space *vm) 3083 { 3084 intel_gmch_remove(); 3085 } 3086 3087 int i915_gem_gtt_init(struct drm_device *dev) 3088 { 3089 struct drm_i915_private *dev_priv = dev->dev_private; 3090 struct i915_gtt *gtt = &dev_priv->gtt; 3091 int ret; 3092 3093 if (INTEL_INFO(dev)->gen <= 5) { 3094 gtt->gtt_probe = i915_gmch_probe; 3095 gtt->base.cleanup = i915_gmch_remove; 3096 } else if (INTEL_INFO(dev)->gen < 8) { 3097 gtt->gtt_probe = gen6_gmch_probe; 3098 gtt->base.cleanup = gen6_gmch_remove; 3099 if (IS_HASWELL(dev) && dev_priv->ellc_size) 3100 gtt->base.pte_encode = iris_pte_encode; 3101 else if (IS_HASWELL(dev)) 3102 gtt->base.pte_encode = hsw_pte_encode; 3103 else if (IS_VALLEYVIEW(dev)) 3104 gtt->base.pte_encode = byt_pte_encode; 3105 else if (INTEL_INFO(dev)->gen >= 7) 3106 gtt->base.pte_encode = ivb_pte_encode; 3107 else 3108 gtt->base.pte_encode = snb_pte_encode; 3109 } else { 3110 dev_priv->gtt.gtt_probe = gen8_gmch_probe; 3111 dev_priv->gtt.base.cleanup = gen6_gmch_remove; 3112 } 3113 3114 gtt->base.dev = dev; 3115 3116 ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, 3117 >t->mappable_base, >t->mappable_end); 3118 if (ret) 3119 return ret; 3120 3121 /* GMADR is the PCI mmio aperture into the global GTT. */ 3122 DRM_INFO("Memory usable by graphics device = %lluM\n", 3123 gtt->base.total >> 20); 3124 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); 3125 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); 3126 #ifdef CONFIG_INTEL_IOMMU 3127 if (intel_iommu_gfx_mapped) 3128 DRM_INFO("VT-d active for gfx access\n"); 3129 #endif 3130 /* 3131 * i915.enable_ppgtt is read-only, so do an early pass to validate the 3132 * user's requested state against the hardware/driver capabilities. We 3133 * do this now so that we can print out any log messages once rather 3134 * than every time we check intel_enable_ppgtt(). 3135 */ 3136 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); 3137 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 3138 3139 return 0; 3140 } 3141 3142 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 3143 { 3144 struct drm_i915_private *dev_priv = dev->dev_private; 3145 struct drm_i915_gem_object *obj; 3146 struct i915_address_space *vm; 3147 struct i915_vma *vma; 3148 bool flush; 3149 3150 i915_check_and_clear_faults(dev); 3151 3152 /* First fill our portion of the GTT with scratch pages */ 3153 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 3154 dev_priv->gtt.base.start, 3155 dev_priv->gtt.base.total, 3156 true); 3157 3158 /* Cache flush objects bound into GGTT and rebind them. */ 3159 vm = &dev_priv->gtt.base; 3160 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 3161 flush = false; 3162 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3163 if (vma->vm != vm) 3164 continue; 3165 3166 WARN_ON(i915_vma_bind(vma, obj->cache_level, 3167 PIN_UPDATE)); 3168 3169 flush = true; 3170 } 3171 3172 if (flush) 3173 i915_gem_clflush_object(obj, obj->pin_display); 3174 } 3175 3176 if (INTEL_INFO(dev)->gen >= 8) { 3177 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3178 chv_setup_private_ppat(dev_priv); 3179 else 3180 bdw_setup_private_ppat(dev_priv); 3181 3182 return; 3183 } 3184 3185 if (USES_PPGTT(dev)) { 3186 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3187 /* TODO: Perhaps it shouldn't be gen6 specific */ 3188 3189 struct i915_hw_ppgtt *ppgtt = 3190 container_of(vm, struct i915_hw_ppgtt, 3191 base); 3192 3193 if (i915_is_ggtt(vm)) 3194 ppgtt = dev_priv->mm.aliasing_ppgtt; 3195 3196 gen6_write_page_range(dev_priv, &ppgtt->pd, 3197 0, ppgtt->base.total); 3198 } 3199 } 3200 3201 i915_ggtt_flush(dev_priv); 3202 } 3203 3204 static struct i915_vma * 3205 __i915_gem_vma_create(struct drm_i915_gem_object *obj, 3206 struct i915_address_space *vm, 3207 const struct i915_ggtt_view *ggtt_view) 3208 { 3209 struct i915_vma *vma; 3210 3211 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 3212 return ERR_PTR(-EINVAL); 3213 3214 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); 3215 if (vma == NULL) 3216 return ERR_PTR(-ENOMEM); 3217 3218 INIT_LIST_HEAD(&vma->vma_link); 3219 INIT_LIST_HEAD(&vma->mm_list); 3220 INIT_LIST_HEAD(&vma->exec_list); 3221 vma->vm = vm; 3222 vma->obj = obj; 3223 3224 if (i915_is_ggtt(vm)) 3225 vma->ggtt_view = *ggtt_view; 3226 3227 list_add_tail(&vma->vma_link, &obj->vma_list); 3228 if (!i915_is_ggtt(vm)) 3229 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 3230 3231 return vma; 3232 } 3233 3234 struct i915_vma * 3235 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 3236 struct i915_address_space *vm) 3237 { 3238 struct i915_vma *vma; 3239 3240 vma = i915_gem_obj_to_vma(obj, vm); 3241 if (!vma) 3242 vma = __i915_gem_vma_create(obj, vm, 3243 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); 3244 3245 return vma; 3246 } 3247 3248 struct i915_vma * 3249 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, 3250 const struct i915_ggtt_view *view) 3251 { 3252 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 3253 struct i915_vma *vma; 3254 3255 if (WARN_ON(!view)) 3256 return ERR_PTR(-EINVAL); 3257 3258 vma = i915_gem_obj_to_ggtt_view(obj, view); 3259 3260 if (IS_ERR(vma)) 3261 return vma; 3262 3263 if (!vma) 3264 vma = __i915_gem_vma_create(obj, ggtt, view); 3265 3266 return vma; 3267 3268 } 3269 3270 static struct scatterlist * 3271 rotate_pages(dma_addr_t *in, unsigned int offset, 3272 unsigned int width, unsigned int height, 3273 struct sg_table *st, struct scatterlist *sg) 3274 { 3275 unsigned int column, row; 3276 unsigned int src_idx; 3277 3278 if (!sg) { 3279 st->nents = 0; 3280 sg = st->sgl; 3281 } 3282 3283 for (column = 0; column < width; column++) { 3284 src_idx = width * (height - 1) + column; 3285 for (row = 0; row < height; row++) { 3286 st->nents++; 3287 /* We don't need the pages, but need to initialize 3288 * the entries so the sg list can be happily traversed. 3289 * The only thing we need are DMA addresses. 3290 */ 3291 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3292 sg_dma_address(sg) = in[offset + src_idx]; 3293 sg_dma_len(sg) = PAGE_SIZE; 3294 sg = sg_next(sg); 3295 src_idx -= width; 3296 } 3297 } 3298 3299 return sg; 3300 } 3301 3302 static struct sg_table * 3303 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, 3304 struct drm_i915_gem_object *obj) 3305 { 3306 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; 3307 unsigned int size_pages = rot_info->size >> PAGE_SHIFT; 3308 unsigned int size_pages_uv; 3309 struct sg_page_iter sg_iter; 3310 unsigned long i; 3311 dma_addr_t *page_addr_list; 3312 struct sg_table *st; 3313 unsigned int uv_start_page; 3314 struct scatterlist *sg; 3315 int ret = -ENOMEM; 3316 3317 /* Allocate a temporary list of source pages for random access. */ 3318 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, 3319 sizeof(dma_addr_t)); 3320 if (!page_addr_list) 3321 return ERR_PTR(ret); 3322 3323 /* Account for UV plane with NV12. */ 3324 if (rot_info->pixel_format == DRM_FORMAT_NV12) 3325 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; 3326 else 3327 size_pages_uv = 0; 3328 3329 /* Allocate target SG list. */ 3330 st = kmalloc(sizeof(*st), GFP_KERNEL); 3331 if (!st) 3332 goto err_st_alloc; 3333 3334 ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); 3335 if (ret) 3336 goto err_sg_alloc; 3337 3338 /* Populate source page list from the object. */ 3339 i = 0; 3340 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 3341 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); 3342 i++; 3343 } 3344 3345 /* Rotate the pages. */ 3346 sg = rotate_pages(page_addr_list, 0, 3347 rot_info->width_pages, rot_info->height_pages, 3348 st, NULL); 3349 3350 /* Append the UV plane if NV12. */ 3351 if (rot_info->pixel_format == DRM_FORMAT_NV12) { 3352 uv_start_page = size_pages; 3353 3354 /* Check for tile-row un-alignment. */ 3355 if (offset_in_page(rot_info->uv_offset)) 3356 uv_start_page--; 3357 3358 rot_info->uv_start_page = uv_start_page; 3359 3360 rotate_pages(page_addr_list, uv_start_page, 3361 rot_info->width_pages_uv, 3362 rot_info->height_pages_uv, 3363 st, sg); 3364 } 3365 3366 DRM_DEBUG_KMS( 3367 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", 3368 obj->base.size, rot_info->pitch, rot_info->height, 3369 rot_info->pixel_format, rot_info->width_pages, 3370 rot_info->height_pages, size_pages + size_pages_uv, 3371 size_pages); 3372 3373 drm_free_large(page_addr_list); 3374 3375 return st; 3376 3377 err_sg_alloc: 3378 kfree(st); 3379 err_st_alloc: 3380 drm_free_large(page_addr_list); 3381 3382 DRM_DEBUG_KMS( 3383 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", 3384 obj->base.size, ret, rot_info->pitch, rot_info->height, 3385 rot_info->pixel_format, rot_info->width_pages, 3386 rot_info->height_pages, size_pages + size_pages_uv, 3387 size_pages); 3388 return ERR_PTR(ret); 3389 } 3390 3391 static struct sg_table * 3392 intel_partial_pages(const struct i915_ggtt_view *view, 3393 struct drm_i915_gem_object *obj) 3394 { 3395 struct sg_table *st; 3396 struct scatterlist *sg; 3397 struct sg_page_iter obj_sg_iter; 3398 int ret = -ENOMEM; 3399 3400 st = kmalloc(sizeof(*st), GFP_KERNEL); 3401 if (!st) 3402 goto err_st_alloc; 3403 3404 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); 3405 if (ret) 3406 goto err_sg_alloc; 3407 3408 sg = st->sgl; 3409 st->nents = 0; 3410 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, 3411 view->params.partial.offset) 3412 { 3413 if (st->nents >= view->params.partial.size) 3414 break; 3415 3416 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3417 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); 3418 sg_dma_len(sg) = PAGE_SIZE; 3419 3420 sg = sg_next(sg); 3421 st->nents++; 3422 } 3423 3424 return st; 3425 3426 err_sg_alloc: 3427 kfree(st); 3428 err_st_alloc: 3429 return ERR_PTR(ret); 3430 } 3431 3432 static int 3433 i915_get_ggtt_vma_pages(struct i915_vma *vma) 3434 { 3435 int ret = 0; 3436 3437 if (vma->ggtt_view.pages) 3438 return 0; 3439 3440 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 3441 vma->ggtt_view.pages = vma->obj->pages; 3442 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 3443 vma->ggtt_view.pages = 3444 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); 3445 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) 3446 vma->ggtt_view.pages = 3447 intel_partial_pages(&vma->ggtt_view, vma->obj); 3448 else 3449 WARN_ONCE(1, "GGTT view %u not implemented!\n", 3450 vma->ggtt_view.type); 3451 3452 if (!vma->ggtt_view.pages) { 3453 DRM_ERROR("Failed to get pages for GGTT view type %u!\n", 3454 vma->ggtt_view.type); 3455 ret = -EINVAL; 3456 } else if (IS_ERR(vma->ggtt_view.pages)) { 3457 ret = PTR_ERR(vma->ggtt_view.pages); 3458 vma->ggtt_view.pages = NULL; 3459 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 3460 vma->ggtt_view.type, ret); 3461 } 3462 3463 return ret; 3464 } 3465 3466 /** 3467 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 3468 * @vma: VMA to map 3469 * @cache_level: mapping cache level 3470 * @flags: flags like global or local mapping 3471 * 3472 * DMA addresses are taken from the scatter-gather table of this object (or of 3473 * this VMA in case of non-default GGTT views) and PTE entries set up. 3474 * Note that DMA addresses are also the only part of the SG table we care about. 3475 */ 3476 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 3477 u32 flags) 3478 { 3479 int ret; 3480 u32 bind_flags; 3481 3482 if (WARN_ON(flags == 0)) 3483 return -EINVAL; 3484 3485 bind_flags = 0; 3486 if (flags & PIN_GLOBAL) 3487 bind_flags |= GLOBAL_BIND; 3488 if (flags & PIN_USER) 3489 bind_flags |= LOCAL_BIND; 3490 3491 if (flags & PIN_UPDATE) 3492 bind_flags |= vma->bound; 3493 else 3494 bind_flags &= ~vma->bound; 3495 3496 if (bind_flags == 0) 3497 return 0; 3498 3499 if (vma->bound == 0 && vma->vm->allocate_va_range) { 3500 trace_i915_va_alloc(vma->vm, 3501 vma->node.start, 3502 vma->node.size, 3503 VM_TO_TRACE_NAME(vma->vm)); 3504 3505 /* XXX: i915_vma_pin() will fix this +- hack */ 3506 vma->pin_count++; 3507 ret = vma->vm->allocate_va_range(vma->vm, 3508 vma->node.start, 3509 vma->node.size); 3510 vma->pin_count--; 3511 if (ret) 3512 return ret; 3513 } 3514 3515 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 3516 if (ret) 3517 return ret; 3518 3519 vma->bound |= bind_flags; 3520 3521 return 0; 3522 } 3523 3524 /** 3525 * i915_ggtt_view_size - Get the size of a GGTT view. 3526 * @obj: Object the view is of. 3527 * @view: The view in question. 3528 * 3529 * @return The size of the GGTT view in bytes. 3530 */ 3531 size_t 3532 i915_ggtt_view_size(struct drm_i915_gem_object *obj, 3533 const struct i915_ggtt_view *view) 3534 { 3535 if (view->type == I915_GGTT_VIEW_NORMAL) { 3536 return obj->base.size; 3537 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 3538 return view->rotation_info.size; 3539 } else if (view->type == I915_GGTT_VIEW_PARTIAL) { 3540 return view->params.partial.size << PAGE_SHIFT; 3541 } else { 3542 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type); 3543 return obj->base.size; 3544 } 3545 } 3546