1 /* 2 * Copyright © 2010 Daniel Vetter 3 * Copyright © 2011-2014 Intel Corporation 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/seq_file.h> 27 #include <linux/stop_machine.h> 28 #include <drm/drmP.h> 29 #include <drm/i915_drm.h> 30 #include "i915_drv.h" 31 #include "i915_vgpu.h" 32 #include "i915_trace.h" 33 #include "intel_drv.h" 34 35 /** 36 * DOC: Global GTT views 37 * 38 * Background and previous state 39 * 40 * Historically objects could exists (be bound) in global GTT space only as 41 * singular instances with a view representing all of the object's backing pages 42 * in a linear fashion. This view will be called a normal view. 43 * 44 * To support multiple views of the same object, where the number of mapped 45 * pages is not equal to the backing store, or where the layout of the pages 46 * is not linear, concept of a GGTT view was added. 47 * 48 * One example of an alternative view is a stereo display driven by a single 49 * image. In this case we would have a framebuffer looking like this 50 * (2x2 pages): 51 * 52 * 12 53 * 34 54 * 55 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 56 * rendering. In contrast, fed to the display engine would be an alternative 57 * view which could look something like this: 58 * 59 * 1212 60 * 3434 61 * 62 * In this example both the size and layout of pages in the alternative view is 63 * different from the normal view. 64 * 65 * Implementation and usage 66 * 67 * GGTT views are implemented using VMAs and are distinguished via enum 68 * i915_ggtt_view_type and struct i915_ggtt_view. 69 * 70 * A new flavour of core GEM functions which work with GGTT bound objects were 71 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 72 * renaming in large amounts of code. They take the struct i915_ggtt_view 73 * parameter encapsulating all metadata required to implement a view. 74 * 75 * As a helper for callers which are only interested in the normal view, 76 * globally const i915_ggtt_view_normal singleton instance exists. All old core 77 * GEM API functions, the ones not taking the view parameter, are operating on, 78 * or with the normal GGTT view. 79 * 80 * Code wanting to add or use a new GGTT view needs to: 81 * 82 * 1. Add a new enum with a suitable name. 83 * 2. Extend the metadata in the i915_ggtt_view structure if required. 84 * 3. Add support to i915_get_vma_pages(). 85 * 86 * New views are required to build a scatter-gather table from within the 87 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 88 * exists for the lifetime of an VMA. 89 * 90 * Core API is designed to have copy semantics which means that passed in 91 * struct i915_ggtt_view does not need to be persistent (left around after 92 * calling the core API functions). 93 * 94 */ 95 96 static int 97 i915_get_ggtt_vma_pages(struct i915_vma *vma); 98 99 const struct i915_ggtt_view i915_ggtt_view_normal = { 100 .type = I915_GGTT_VIEW_NORMAL, 101 }; 102 const struct i915_ggtt_view i915_ggtt_view_rotated = { 103 .type = I915_GGTT_VIEW_ROTATED, 104 }; 105 106 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) 107 { 108 bool has_aliasing_ppgtt; 109 bool has_full_ppgtt; 110 bool has_full_48bit_ppgtt; 111 112 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; 113 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; 114 has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9; 115 116 if (intel_vgpu_active(dev)) 117 has_full_ppgtt = false; /* emulation is too hard */ 118 119 /* 120 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 121 * execlists, the sole mechanism available to submit work. 122 */ 123 if (INTEL_INFO(dev)->gen < 9 && 124 (enable_ppgtt == 0 || !has_aliasing_ppgtt)) 125 return 0; 126 127 if (enable_ppgtt == 1) 128 return 1; 129 130 if (enable_ppgtt == 2 && has_full_ppgtt) 131 return 2; 132 133 if (enable_ppgtt == 3 && has_full_48bit_ppgtt) 134 return 3; 135 136 #ifdef CONFIG_INTEL_IOMMU 137 /* Disable ppgtt on SNB if VT-d is on. */ 138 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { 139 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 140 return 0; 141 } 142 #endif 143 144 /* Early VLV doesn't have this */ 145 if (IS_VALLEYVIEW(dev) && dev->pdev->revision < 0xb) { 146 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 147 return 0; 148 } 149 150 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) 151 return has_full_48bit_ppgtt ? 3 : 2; 152 else 153 return has_aliasing_ppgtt ? 1 : 0; 154 } 155 156 static int ppgtt_bind_vma(struct i915_vma *vma, 157 enum i915_cache_level cache_level, 158 u32 unused) 159 { 160 u32 pte_flags = 0; 161 162 /* Currently applicable only to VLV */ 163 if (vma->obj->gt_ro) 164 pte_flags |= PTE_READ_ONLY; 165 166 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, 167 cache_level, pte_flags); 168 169 return 0; 170 } 171 172 static void ppgtt_unbind_vma(struct i915_vma *vma) 173 { 174 vma->vm->clear_range(vma->vm, 175 vma->node.start, 176 vma->obj->base.size, 177 true); 178 } 179 180 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 181 enum i915_cache_level level, 182 bool valid) 183 { 184 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; 185 pte |= addr; 186 187 switch (level) { 188 case I915_CACHE_NONE: 189 pte |= PPAT_UNCACHED_INDEX; 190 break; 191 case I915_CACHE_WT: 192 pte |= PPAT_DISPLAY_ELLC_INDEX; 193 break; 194 default: 195 pte |= PPAT_CACHED_INDEX; 196 break; 197 } 198 199 return pte; 200 } 201 202 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 203 const enum i915_cache_level level) 204 { 205 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 206 pde |= addr; 207 if (level != I915_CACHE_NONE) 208 pde |= PPAT_CACHED_PDE_INDEX; 209 else 210 pde |= PPAT_UNCACHED_INDEX; 211 return pde; 212 } 213 214 #define gen8_pdpe_encode gen8_pde_encode 215 #define gen8_pml4e_encode gen8_pde_encode 216 217 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 218 enum i915_cache_level level, 219 bool valid, u32 unused) 220 { 221 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 222 pte |= GEN6_PTE_ADDR_ENCODE(addr); 223 224 switch (level) { 225 case I915_CACHE_L3_LLC: 226 case I915_CACHE_LLC: 227 pte |= GEN6_PTE_CACHE_LLC; 228 break; 229 case I915_CACHE_NONE: 230 pte |= GEN6_PTE_UNCACHED; 231 break; 232 default: 233 MISSING_CASE(level); 234 } 235 236 return pte; 237 } 238 239 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 240 enum i915_cache_level level, 241 bool valid, u32 unused) 242 { 243 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 244 pte |= GEN6_PTE_ADDR_ENCODE(addr); 245 246 switch (level) { 247 case I915_CACHE_L3_LLC: 248 pte |= GEN7_PTE_CACHE_L3_LLC; 249 break; 250 case I915_CACHE_LLC: 251 pte |= GEN6_PTE_CACHE_LLC; 252 break; 253 case I915_CACHE_NONE: 254 pte |= GEN6_PTE_UNCACHED; 255 break; 256 default: 257 MISSING_CASE(level); 258 } 259 260 return pte; 261 } 262 263 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 264 enum i915_cache_level level, 265 bool valid, u32 flags) 266 { 267 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 268 pte |= GEN6_PTE_ADDR_ENCODE(addr); 269 270 if (!(flags & PTE_READ_ONLY)) 271 pte |= BYT_PTE_WRITEABLE; 272 273 if (level != I915_CACHE_NONE) 274 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 275 276 return pte; 277 } 278 279 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 280 enum i915_cache_level level, 281 bool valid, u32 unused) 282 { 283 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 284 pte |= HSW_PTE_ADDR_ENCODE(addr); 285 286 if (level != I915_CACHE_NONE) 287 pte |= HSW_WB_LLC_AGE3; 288 289 return pte; 290 } 291 292 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 293 enum i915_cache_level level, 294 bool valid, u32 unused) 295 { 296 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 297 pte |= HSW_PTE_ADDR_ENCODE(addr); 298 299 switch (level) { 300 case I915_CACHE_NONE: 301 break; 302 case I915_CACHE_WT: 303 pte |= HSW_WT_ELLC_LLC_AGE3; 304 break; 305 default: 306 pte |= HSW_WB_ELLC_LLC_AGE3; 307 break; 308 } 309 310 return pte; 311 } 312 313 static int __setup_page_dma(struct drm_device *dev, 314 struct i915_page_dma *p, gfp_t flags) 315 { 316 struct device *device = &dev->pdev->dev; 317 318 p->page = alloc_page(flags); 319 if (!p->page) 320 return -ENOMEM; 321 322 p->daddr = dma_map_page(device, 323 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); 324 325 if (dma_mapping_error(device, p->daddr)) { 326 __free_page(p->page); 327 return -EINVAL; 328 } 329 330 return 0; 331 } 332 333 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 334 { 335 return __setup_page_dma(dev, p, GFP_KERNEL); 336 } 337 338 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 339 { 340 if (WARN_ON(!p->page)) 341 return; 342 343 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); 344 __free_page(p->page); 345 memset(p, 0, sizeof(*p)); 346 } 347 348 static void *kmap_page_dma(struct i915_page_dma *p) 349 { 350 return kmap_atomic(p->page); 351 } 352 353 /* We use the flushing unmap only with ppgtt structures: 354 * page directories, page tables and scratch pages. 355 */ 356 static void kunmap_page_dma(struct drm_device *dev, void *vaddr) 357 { 358 /* There are only few exceptions for gen >=6. chv and bxt. 359 * And we are not sure about the latter so play safe for now. 360 */ 361 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 362 drm_clflush_virt_range(vaddr, PAGE_SIZE); 363 364 kunmap_atomic(vaddr); 365 } 366 367 #define kmap_px(px) kmap_page_dma(px_base(px)) 368 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) 369 370 #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 371 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 372 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) 373 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) 374 375 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, 376 const uint64_t val) 377 { 378 int i; 379 uint64_t * const vaddr = kmap_page_dma(p); 380 381 for (i = 0; i < 512; i++) 382 vaddr[i] = val; 383 384 kunmap_page_dma(dev, vaddr); 385 } 386 387 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, 388 const uint32_t val32) 389 { 390 uint64_t v = val32; 391 392 v = v << 32 | val32; 393 394 fill_page_dma(dev, p, v); 395 } 396 397 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) 398 { 399 struct i915_page_scratch *sp; 400 int ret; 401 402 sp = kzalloc(sizeof(*sp), GFP_KERNEL); 403 if (sp == NULL) 404 return ERR_PTR(-ENOMEM); 405 406 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); 407 if (ret) { 408 kfree(sp); 409 return ERR_PTR(ret); 410 } 411 412 set_pages_uc(px_page(sp), 1); 413 414 return sp; 415 } 416 417 static void free_scratch_page(struct drm_device *dev, 418 struct i915_page_scratch *sp) 419 { 420 set_pages_wb(px_page(sp), 1); 421 422 cleanup_px(dev, sp); 423 kfree(sp); 424 } 425 426 static struct i915_page_table *alloc_pt(struct drm_device *dev) 427 { 428 struct i915_page_table *pt; 429 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 430 GEN8_PTES : GEN6_PTES; 431 int ret = -ENOMEM; 432 433 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 434 if (!pt) 435 return ERR_PTR(-ENOMEM); 436 437 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), 438 GFP_KERNEL); 439 440 if (!pt->used_ptes) 441 goto fail_bitmap; 442 443 ret = setup_px(dev, pt); 444 if (ret) 445 goto fail_page_m; 446 447 return pt; 448 449 fail_page_m: 450 kfree(pt->used_ptes); 451 fail_bitmap: 452 kfree(pt); 453 454 return ERR_PTR(ret); 455 } 456 457 static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 458 { 459 cleanup_px(dev, pt); 460 kfree(pt->used_ptes); 461 kfree(pt); 462 } 463 464 static void gen8_initialize_pt(struct i915_address_space *vm, 465 struct i915_page_table *pt) 466 { 467 gen8_pte_t scratch_pte; 468 469 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 470 I915_CACHE_LLC, true); 471 472 fill_px(vm->dev, pt, scratch_pte); 473 } 474 475 static void gen6_initialize_pt(struct i915_address_space *vm, 476 struct i915_page_table *pt) 477 { 478 gen6_pte_t scratch_pte; 479 480 WARN_ON(px_dma(vm->scratch_page) == 0); 481 482 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 483 I915_CACHE_LLC, true, 0); 484 485 fill32_px(vm->dev, pt, scratch_pte); 486 } 487 488 static struct i915_page_directory *alloc_pd(struct drm_device *dev) 489 { 490 struct i915_page_directory *pd; 491 int ret = -ENOMEM; 492 493 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 494 if (!pd) 495 return ERR_PTR(-ENOMEM); 496 497 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), 498 sizeof(*pd->used_pdes), GFP_KERNEL); 499 if (!pd->used_pdes) 500 goto fail_bitmap; 501 502 ret = setup_px(dev, pd); 503 if (ret) 504 goto fail_page_m; 505 506 return pd; 507 508 fail_page_m: 509 kfree(pd->used_pdes); 510 fail_bitmap: 511 kfree(pd); 512 513 return ERR_PTR(ret); 514 } 515 516 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 517 { 518 if (px_page(pd)) { 519 cleanup_px(dev, pd); 520 kfree(pd->used_pdes); 521 kfree(pd); 522 } 523 } 524 525 static void gen8_initialize_pd(struct i915_address_space *vm, 526 struct i915_page_directory *pd) 527 { 528 gen8_pde_t scratch_pde; 529 530 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); 531 532 fill_px(vm->dev, pd, scratch_pde); 533 } 534 535 static int __pdp_init(struct drm_device *dev, 536 struct i915_page_directory_pointer *pdp) 537 { 538 size_t pdpes = I915_PDPES_PER_PDP(dev); 539 540 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), 541 sizeof(unsigned long), 542 GFP_KERNEL); 543 if (!pdp->used_pdpes) 544 return -ENOMEM; 545 546 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), 547 GFP_KERNEL); 548 if (!pdp->page_directory) { 549 kfree(pdp->used_pdpes); 550 /* the PDP might be the statically allocated top level. Keep it 551 * as clean as possible */ 552 pdp->used_pdpes = NULL; 553 return -ENOMEM; 554 } 555 556 return 0; 557 } 558 559 static void __pdp_fini(struct i915_page_directory_pointer *pdp) 560 { 561 kfree(pdp->used_pdpes); 562 kfree(pdp->page_directory); 563 pdp->page_directory = NULL; 564 } 565 566 static struct 567 i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) 568 { 569 struct i915_page_directory_pointer *pdp; 570 int ret = -ENOMEM; 571 572 WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); 573 574 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 575 if (!pdp) 576 return ERR_PTR(-ENOMEM); 577 578 ret = __pdp_init(dev, pdp); 579 if (ret) 580 goto fail_bitmap; 581 582 ret = setup_px(dev, pdp); 583 if (ret) 584 goto fail_page_m; 585 586 return pdp; 587 588 fail_page_m: 589 __pdp_fini(pdp); 590 fail_bitmap: 591 kfree(pdp); 592 593 return ERR_PTR(ret); 594 } 595 596 static void free_pdp(struct drm_device *dev, 597 struct i915_page_directory_pointer *pdp) 598 { 599 __pdp_fini(pdp); 600 if (USES_FULL_48BIT_PPGTT(dev)) { 601 cleanup_px(dev, pdp); 602 kfree(pdp); 603 } 604 } 605 606 static void gen8_initialize_pdp(struct i915_address_space *vm, 607 struct i915_page_directory_pointer *pdp) 608 { 609 gen8_ppgtt_pdpe_t scratch_pdpe; 610 611 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); 612 613 fill_px(vm->dev, pdp, scratch_pdpe); 614 } 615 616 static void gen8_initialize_pml4(struct i915_address_space *vm, 617 struct i915_pml4 *pml4) 618 { 619 gen8_ppgtt_pml4e_t scratch_pml4e; 620 621 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), 622 I915_CACHE_LLC); 623 624 fill_px(vm->dev, pml4, scratch_pml4e); 625 } 626 627 static void 628 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, 629 struct i915_page_directory_pointer *pdp, 630 struct i915_page_directory *pd, 631 int index) 632 { 633 gen8_ppgtt_pdpe_t *page_directorypo; 634 635 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 636 return; 637 638 page_directorypo = kmap_px(pdp); 639 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); 640 kunmap_px(ppgtt, page_directorypo); 641 } 642 643 static void 644 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, 645 struct i915_pml4 *pml4, 646 struct i915_page_directory_pointer *pdp, 647 int index) 648 { 649 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); 650 651 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); 652 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 653 kunmap_px(ppgtt, pagemap); 654 } 655 656 /* Broadwell Page Directory Pointer Descriptors */ 657 static int gen8_write_pdp(struct drm_i915_gem_request *req, 658 unsigned entry, 659 dma_addr_t addr) 660 { 661 struct intel_engine_cs *ring = req->ring; 662 int ret; 663 664 BUG_ON(entry >= 4); 665 666 ret = intel_ring_begin(req, 6); 667 if (ret) 668 return ret; 669 670 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 671 intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(ring, entry)); 672 intel_ring_emit(ring, upper_32_bits(addr)); 673 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 674 intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(ring, entry)); 675 intel_ring_emit(ring, lower_32_bits(addr)); 676 intel_ring_advance(ring); 677 678 return 0; 679 } 680 681 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, 682 struct drm_i915_gem_request *req) 683 { 684 int i, ret; 685 686 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { 687 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 688 689 ret = gen8_write_pdp(req, i, pd_daddr); 690 if (ret) 691 return ret; 692 } 693 694 return 0; 695 } 696 697 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, 698 struct drm_i915_gem_request *req) 699 { 700 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 701 } 702 703 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, 704 struct i915_page_directory_pointer *pdp, 705 uint64_t start, 706 uint64_t length, 707 gen8_pte_t scratch_pte) 708 { 709 struct i915_hw_ppgtt *ppgtt = 710 container_of(vm, struct i915_hw_ppgtt, base); 711 gen8_pte_t *pt_vaddr; 712 unsigned pdpe = gen8_pdpe_index(start); 713 unsigned pde = gen8_pde_index(start); 714 unsigned pte = gen8_pte_index(start); 715 unsigned num_entries = length >> PAGE_SHIFT; 716 unsigned last_pte, i; 717 718 if (WARN_ON(!pdp)) 719 return; 720 721 while (num_entries) { 722 struct i915_page_directory *pd; 723 struct i915_page_table *pt; 724 725 if (WARN_ON(!pdp->page_directory[pdpe])) 726 break; 727 728 pd = pdp->page_directory[pdpe]; 729 730 if (WARN_ON(!pd->page_table[pde])) 731 break; 732 733 pt = pd->page_table[pde]; 734 735 if (WARN_ON(!px_page(pt))) 736 break; 737 738 last_pte = pte + num_entries; 739 if (last_pte > GEN8_PTES) 740 last_pte = GEN8_PTES; 741 742 pt_vaddr = kmap_px(pt); 743 744 for (i = pte; i < last_pte; i++) { 745 pt_vaddr[i] = scratch_pte; 746 num_entries--; 747 } 748 749 kunmap_px(ppgtt, pt); 750 751 pte = 0; 752 if (++pde == I915_PDES) { 753 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 754 break; 755 pde = 0; 756 } 757 } 758 } 759 760 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, 761 uint64_t start, 762 uint64_t length, 763 bool use_scratch) 764 { 765 struct i915_hw_ppgtt *ppgtt = 766 container_of(vm, struct i915_hw_ppgtt, base); 767 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 768 I915_CACHE_LLC, use_scratch); 769 770 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 771 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, 772 scratch_pte); 773 } else { 774 uint64_t pml4e; 775 struct i915_page_directory_pointer *pdp; 776 777 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) { 778 gen8_ppgtt_clear_pte_range(vm, pdp, start, length, 779 scratch_pte); 780 } 781 } 782 } 783 784 static void 785 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, 786 struct i915_page_directory_pointer *pdp, 787 struct sg_page_iter *sg_iter, 788 uint64_t start, 789 enum i915_cache_level cache_level) 790 { 791 struct i915_hw_ppgtt *ppgtt = 792 container_of(vm, struct i915_hw_ppgtt, base); 793 gen8_pte_t *pt_vaddr; 794 unsigned pdpe = gen8_pdpe_index(start); 795 unsigned pde = gen8_pde_index(start); 796 unsigned pte = gen8_pte_index(start); 797 798 pt_vaddr = NULL; 799 800 while (__sg_page_iter_next(sg_iter)) { 801 if (pt_vaddr == NULL) { 802 struct i915_page_directory *pd = pdp->page_directory[pdpe]; 803 struct i915_page_table *pt = pd->page_table[pde]; 804 pt_vaddr = kmap_px(pt); 805 } 806 807 pt_vaddr[pte] = 808 gen8_pte_encode(sg_page_iter_dma_address(sg_iter), 809 cache_level, true); 810 if (++pte == GEN8_PTES) { 811 kunmap_px(ppgtt, pt_vaddr); 812 pt_vaddr = NULL; 813 if (++pde == I915_PDES) { 814 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 815 break; 816 pde = 0; 817 } 818 pte = 0; 819 } 820 } 821 822 if (pt_vaddr) 823 kunmap_px(ppgtt, pt_vaddr); 824 } 825 826 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 827 struct sg_table *pages, 828 uint64_t start, 829 enum i915_cache_level cache_level, 830 u32 unused) 831 { 832 struct i915_hw_ppgtt *ppgtt = 833 container_of(vm, struct i915_hw_ppgtt, base); 834 struct sg_page_iter sg_iter; 835 836 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); 837 838 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 839 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, 840 cache_level); 841 } else { 842 struct i915_page_directory_pointer *pdp; 843 uint64_t pml4e; 844 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; 845 846 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) { 847 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, 848 start, cache_level); 849 } 850 } 851 } 852 853 static void gen8_free_page_tables(struct drm_device *dev, 854 struct i915_page_directory *pd) 855 { 856 int i; 857 858 if (!px_page(pd)) 859 return; 860 861 for_each_set_bit(i, pd->used_pdes, I915_PDES) { 862 if (WARN_ON(!pd->page_table[i])) 863 continue; 864 865 free_pt(dev, pd->page_table[i]); 866 pd->page_table[i] = NULL; 867 } 868 } 869 870 static int gen8_init_scratch(struct i915_address_space *vm) 871 { 872 struct drm_device *dev = vm->dev; 873 874 vm->scratch_page = alloc_scratch_page(dev); 875 if (IS_ERR(vm->scratch_page)) 876 return PTR_ERR(vm->scratch_page); 877 878 vm->scratch_pt = alloc_pt(dev); 879 if (IS_ERR(vm->scratch_pt)) { 880 free_scratch_page(dev, vm->scratch_page); 881 return PTR_ERR(vm->scratch_pt); 882 } 883 884 vm->scratch_pd = alloc_pd(dev); 885 if (IS_ERR(vm->scratch_pd)) { 886 free_pt(dev, vm->scratch_pt); 887 free_scratch_page(dev, vm->scratch_page); 888 return PTR_ERR(vm->scratch_pd); 889 } 890 891 if (USES_FULL_48BIT_PPGTT(dev)) { 892 vm->scratch_pdp = alloc_pdp(dev); 893 if (IS_ERR(vm->scratch_pdp)) { 894 free_pd(dev, vm->scratch_pd); 895 free_pt(dev, vm->scratch_pt); 896 free_scratch_page(dev, vm->scratch_page); 897 return PTR_ERR(vm->scratch_pdp); 898 } 899 } 900 901 gen8_initialize_pt(vm, vm->scratch_pt); 902 gen8_initialize_pd(vm, vm->scratch_pd); 903 if (USES_FULL_48BIT_PPGTT(dev)) 904 gen8_initialize_pdp(vm, vm->scratch_pdp); 905 906 return 0; 907 } 908 909 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) 910 { 911 enum vgt_g2v_type msg; 912 struct drm_device *dev = ppgtt->base.dev; 913 struct drm_i915_private *dev_priv = dev->dev_private; 914 int i; 915 916 if (USES_FULL_48BIT_PPGTT(dev)) { 917 u64 daddr = px_dma(&ppgtt->pml4); 918 919 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); 920 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); 921 922 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : 923 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); 924 } else { 925 for (i = 0; i < GEN8_LEGACY_PDPES; i++) { 926 u64 daddr = i915_page_dir_dma_addr(ppgtt, i); 927 928 I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr)); 929 I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr)); 930 } 931 932 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : 933 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); 934 } 935 936 I915_WRITE(vgtif_reg(g2v_notify), msg); 937 938 return 0; 939 } 940 941 static void gen8_free_scratch(struct i915_address_space *vm) 942 { 943 struct drm_device *dev = vm->dev; 944 945 if (USES_FULL_48BIT_PPGTT(dev)) 946 free_pdp(dev, vm->scratch_pdp); 947 free_pd(dev, vm->scratch_pd); 948 free_pt(dev, vm->scratch_pt); 949 free_scratch_page(dev, vm->scratch_page); 950 } 951 952 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, 953 struct i915_page_directory_pointer *pdp) 954 { 955 int i; 956 957 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { 958 if (WARN_ON(!pdp->page_directory[i])) 959 continue; 960 961 gen8_free_page_tables(dev, pdp->page_directory[i]); 962 free_pd(dev, pdp->page_directory[i]); 963 } 964 965 free_pdp(dev, pdp); 966 } 967 968 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 969 { 970 int i; 971 972 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { 973 if (WARN_ON(!ppgtt->pml4.pdps[i])) 974 continue; 975 976 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); 977 } 978 979 cleanup_px(ppgtt->base.dev, &ppgtt->pml4); 980 } 981 982 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 983 { 984 struct i915_hw_ppgtt *ppgtt = 985 container_of(vm, struct i915_hw_ppgtt, base); 986 987 if (intel_vgpu_active(vm->dev)) 988 gen8_ppgtt_notify_vgt(ppgtt, false); 989 990 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 991 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); 992 else 993 gen8_ppgtt_cleanup_4lvl(ppgtt); 994 995 gen8_free_scratch(vm); 996 } 997 998 /** 999 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. 1000 * @vm: Master vm structure. 1001 * @pd: Page directory for this address range. 1002 * @start: Starting virtual address to begin allocations. 1003 * @length: Size of the allocations. 1004 * @new_pts: Bitmap set by function with new allocations. Likely used by the 1005 * caller to free on error. 1006 * 1007 * Allocate the required number of page tables. Extremely similar to 1008 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by 1009 * the page directory boundary (instead of the page directory pointer). That 1010 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is 1011 * possible, and likely that the caller will need to use multiple calls of this 1012 * function to achieve the appropriate allocation. 1013 * 1014 * Return: 0 if success; negative error code otherwise. 1015 */ 1016 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, 1017 struct i915_page_directory *pd, 1018 uint64_t start, 1019 uint64_t length, 1020 unsigned long *new_pts) 1021 { 1022 struct drm_device *dev = vm->dev; 1023 struct i915_page_table *pt; 1024 uint32_t pde; 1025 1026 gen8_for_each_pde(pt, pd, start, length, pde) { 1027 /* Don't reallocate page tables */ 1028 if (test_bit(pde, pd->used_pdes)) { 1029 /* Scratch is never allocated this way */ 1030 WARN_ON(pt == vm->scratch_pt); 1031 continue; 1032 } 1033 1034 pt = alloc_pt(dev); 1035 if (IS_ERR(pt)) 1036 goto unwind_out; 1037 1038 gen8_initialize_pt(vm, pt); 1039 pd->page_table[pde] = pt; 1040 __set_bit(pde, new_pts); 1041 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); 1042 } 1043 1044 return 0; 1045 1046 unwind_out: 1047 for_each_set_bit(pde, new_pts, I915_PDES) 1048 free_pt(dev, pd->page_table[pde]); 1049 1050 return -ENOMEM; 1051 } 1052 1053 /** 1054 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. 1055 * @vm: Master vm structure. 1056 * @pdp: Page directory pointer for this address range. 1057 * @start: Starting virtual address to begin allocations. 1058 * @length: Size of the allocations. 1059 * @new_pds: Bitmap set by function with new allocations. Likely used by the 1060 * caller to free on error. 1061 * 1062 * Allocate the required number of page directories starting at the pde index of 1063 * @start, and ending at the pde index @start + @length. This function will skip 1064 * over already allocated page directories within the range, and only allocate 1065 * new ones, setting the appropriate pointer within the pdp as well as the 1066 * correct position in the bitmap @new_pds. 1067 * 1068 * The function will only allocate the pages within the range for a give page 1069 * directory pointer. In other words, if @start + @length straddles a virtually 1070 * addressed PDP boundary (512GB for 4k pages), there will be more allocations 1071 * required by the caller, This is not currently possible, and the BUG in the 1072 * code will prevent it. 1073 * 1074 * Return: 0 if success; negative error code otherwise. 1075 */ 1076 static int 1077 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, 1078 struct i915_page_directory_pointer *pdp, 1079 uint64_t start, 1080 uint64_t length, 1081 unsigned long *new_pds) 1082 { 1083 struct drm_device *dev = vm->dev; 1084 struct i915_page_directory *pd; 1085 uint32_t pdpe; 1086 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1087 1088 WARN_ON(!bitmap_empty(new_pds, pdpes)); 1089 1090 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1091 if (test_bit(pdpe, pdp->used_pdpes)) 1092 continue; 1093 1094 pd = alloc_pd(dev); 1095 if (IS_ERR(pd)) 1096 goto unwind_out; 1097 1098 gen8_initialize_pd(vm, pd); 1099 pdp->page_directory[pdpe] = pd; 1100 __set_bit(pdpe, new_pds); 1101 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); 1102 } 1103 1104 return 0; 1105 1106 unwind_out: 1107 for_each_set_bit(pdpe, new_pds, pdpes) 1108 free_pd(dev, pdp->page_directory[pdpe]); 1109 1110 return -ENOMEM; 1111 } 1112 1113 /** 1114 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. 1115 * @vm: Master vm structure. 1116 * @pml4: Page map level 4 for this address range. 1117 * @start: Starting virtual address to begin allocations. 1118 * @length: Size of the allocations. 1119 * @new_pdps: Bitmap set by function with new allocations. Likely used by the 1120 * caller to free on error. 1121 * 1122 * Allocate the required number of page directory pointers. Extremely similar to 1123 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). 1124 * The main difference is here we are limited by the pml4 boundary (instead of 1125 * the page directory pointer). 1126 * 1127 * Return: 0 if success; negative error code otherwise. 1128 */ 1129 static int 1130 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, 1131 struct i915_pml4 *pml4, 1132 uint64_t start, 1133 uint64_t length, 1134 unsigned long *new_pdps) 1135 { 1136 struct drm_device *dev = vm->dev; 1137 struct i915_page_directory_pointer *pdp; 1138 uint32_t pml4e; 1139 1140 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); 1141 1142 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1143 if (!test_bit(pml4e, pml4->used_pml4es)) { 1144 pdp = alloc_pdp(dev); 1145 if (IS_ERR(pdp)) 1146 goto unwind_out; 1147 1148 gen8_initialize_pdp(vm, pdp); 1149 pml4->pdps[pml4e] = pdp; 1150 __set_bit(pml4e, new_pdps); 1151 trace_i915_page_directory_pointer_entry_alloc(vm, 1152 pml4e, 1153 start, 1154 GEN8_PML4E_SHIFT); 1155 } 1156 } 1157 1158 return 0; 1159 1160 unwind_out: 1161 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1162 free_pdp(dev, pml4->pdps[pml4e]); 1163 1164 return -ENOMEM; 1165 } 1166 1167 static void 1168 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) 1169 { 1170 kfree(new_pts); 1171 kfree(new_pds); 1172 } 1173 1174 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both 1175 * of these are based on the number of PDPEs in the system. 1176 */ 1177 static 1178 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, 1179 unsigned long **new_pts, 1180 uint32_t pdpes) 1181 { 1182 unsigned long *pds; 1183 unsigned long *pts; 1184 1185 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); 1186 if (!pds) 1187 return -ENOMEM; 1188 1189 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), 1190 GFP_TEMPORARY); 1191 if (!pts) 1192 goto err_out; 1193 1194 *new_pds = pds; 1195 *new_pts = pts; 1196 1197 return 0; 1198 1199 err_out: 1200 free_gen8_temp_bitmaps(pds, pts); 1201 return -ENOMEM; 1202 } 1203 1204 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 1205 * the page table structures, we mark them dirty so that 1206 * context switching/execlist queuing code takes extra steps 1207 * to ensure that tlbs are flushed. 1208 */ 1209 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 1210 { 1211 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 1212 } 1213 1214 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, 1215 struct i915_page_directory_pointer *pdp, 1216 uint64_t start, 1217 uint64_t length) 1218 { 1219 struct i915_hw_ppgtt *ppgtt = 1220 container_of(vm, struct i915_hw_ppgtt, base); 1221 unsigned long *new_page_dirs, *new_page_tables; 1222 struct drm_device *dev = vm->dev; 1223 struct i915_page_directory *pd; 1224 const uint64_t orig_start = start; 1225 const uint64_t orig_length = length; 1226 uint32_t pdpe; 1227 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1228 int ret; 1229 1230 /* Wrap is never okay since we can only represent 48b, and we don't 1231 * actually use the other side of the canonical address space. 1232 */ 1233 if (WARN_ON(start + length < start)) 1234 return -ENODEV; 1235 1236 if (WARN_ON(start + length > vm->total)) 1237 return -ENODEV; 1238 1239 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1240 if (ret) 1241 return ret; 1242 1243 /* Do the allocations first so we can easily bail out */ 1244 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, 1245 new_page_dirs); 1246 if (ret) { 1247 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1248 return ret; 1249 } 1250 1251 /* For every page directory referenced, allocate page tables */ 1252 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1253 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, 1254 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); 1255 if (ret) 1256 goto err_out; 1257 } 1258 1259 start = orig_start; 1260 length = orig_length; 1261 1262 /* Allocations have completed successfully, so set the bitmaps, and do 1263 * the mappings. */ 1264 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1265 gen8_pde_t *const page_directory = kmap_px(pd); 1266 struct i915_page_table *pt; 1267 uint64_t pd_len = length; 1268 uint64_t pd_start = start; 1269 uint32_t pde; 1270 1271 /* Every pd should be allocated, we just did that above. */ 1272 WARN_ON(!pd); 1273 1274 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) { 1275 /* Same reasoning as pd */ 1276 WARN_ON(!pt); 1277 WARN_ON(!pd_len); 1278 WARN_ON(!gen8_pte_count(pd_start, pd_len)); 1279 1280 /* Set our used ptes within the page table */ 1281 bitmap_set(pt->used_ptes, 1282 gen8_pte_index(pd_start), 1283 gen8_pte_count(pd_start, pd_len)); 1284 1285 /* Our pde is now pointing to the pagetable, pt */ 1286 __set_bit(pde, pd->used_pdes); 1287 1288 /* Map the PDE to the page table */ 1289 page_directory[pde] = gen8_pde_encode(px_dma(pt), 1290 I915_CACHE_LLC); 1291 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, 1292 gen8_pte_index(start), 1293 gen8_pte_count(start, length), 1294 GEN8_PTES); 1295 1296 /* NB: We haven't yet mapped ptes to pages. At this 1297 * point we're still relying on insert_entries() */ 1298 } 1299 1300 kunmap_px(ppgtt, page_directory); 1301 __set_bit(pdpe, pdp->used_pdpes); 1302 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); 1303 } 1304 1305 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1306 mark_tlbs_dirty(ppgtt); 1307 return 0; 1308 1309 err_out: 1310 while (pdpe--) { 1311 unsigned long temp; 1312 1313 for_each_set_bit(temp, new_page_tables + pdpe * 1314 BITS_TO_LONGS(I915_PDES), I915_PDES) 1315 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); 1316 } 1317 1318 for_each_set_bit(pdpe, new_page_dirs, pdpes) 1319 free_pd(dev, pdp->page_directory[pdpe]); 1320 1321 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1322 mark_tlbs_dirty(ppgtt); 1323 return ret; 1324 } 1325 1326 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, 1327 struct i915_pml4 *pml4, 1328 uint64_t start, 1329 uint64_t length) 1330 { 1331 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); 1332 struct i915_hw_ppgtt *ppgtt = 1333 container_of(vm, struct i915_hw_ppgtt, base); 1334 struct i915_page_directory_pointer *pdp; 1335 uint64_t pml4e; 1336 int ret = 0; 1337 1338 /* Do the pml4 allocations first, so we don't need to track the newly 1339 * allocated tables below the pdp */ 1340 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); 1341 1342 /* The pagedirectory and pagetable allocations are done in the shared 3 1343 * and 4 level code. Just allocate the pdps. 1344 */ 1345 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, 1346 new_pdps); 1347 if (ret) 1348 return ret; 1349 1350 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, 1351 "The allocation has spanned more than 512GB. " 1352 "It is highly likely this is incorrect."); 1353 1354 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1355 WARN_ON(!pdp); 1356 1357 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); 1358 if (ret) 1359 goto err_out; 1360 1361 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); 1362 } 1363 1364 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, 1365 GEN8_PML4ES_PER_PML4); 1366 1367 return 0; 1368 1369 err_out: 1370 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1371 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); 1372 1373 return ret; 1374 } 1375 1376 static int gen8_alloc_va_range(struct i915_address_space *vm, 1377 uint64_t start, uint64_t length) 1378 { 1379 struct i915_hw_ppgtt *ppgtt = 1380 container_of(vm, struct i915_hw_ppgtt, base); 1381 1382 if (USES_FULL_48BIT_PPGTT(vm->dev)) 1383 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); 1384 else 1385 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); 1386 } 1387 1388 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, 1389 uint64_t start, uint64_t length, 1390 gen8_pte_t scratch_pte, 1391 struct seq_file *m) 1392 { 1393 struct i915_page_directory *pd; 1394 uint32_t pdpe; 1395 1396 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { 1397 struct i915_page_table *pt; 1398 uint64_t pd_len = length; 1399 uint64_t pd_start = start; 1400 uint32_t pde; 1401 1402 if (!test_bit(pdpe, pdp->used_pdpes)) 1403 continue; 1404 1405 seq_printf(m, "\tPDPE #%d\n", pdpe); 1406 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) { 1407 uint32_t pte; 1408 gen8_pte_t *pt_vaddr; 1409 1410 if (!test_bit(pde, pd->used_pdes)) 1411 continue; 1412 1413 pt_vaddr = kmap_px(pt); 1414 for (pte = 0; pte < GEN8_PTES; pte += 4) { 1415 uint64_t va = 1416 (pdpe << GEN8_PDPE_SHIFT) | 1417 (pde << GEN8_PDE_SHIFT) | 1418 (pte << GEN8_PTE_SHIFT); 1419 int i; 1420 bool found = false; 1421 1422 for (i = 0; i < 4; i++) 1423 if (pt_vaddr[pte + i] != scratch_pte) 1424 found = true; 1425 if (!found) 1426 continue; 1427 1428 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); 1429 for (i = 0; i < 4; i++) { 1430 if (pt_vaddr[pte + i] != scratch_pte) 1431 seq_printf(m, " %llx", pt_vaddr[pte + i]); 1432 else 1433 seq_puts(m, " SCRATCH "); 1434 } 1435 seq_puts(m, "\n"); 1436 } 1437 /* don't use kunmap_px, it could trigger 1438 * an unnecessary flush. 1439 */ 1440 kunmap_atomic(pt_vaddr); 1441 } 1442 } 1443 } 1444 1445 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1446 { 1447 struct i915_address_space *vm = &ppgtt->base; 1448 uint64_t start = ppgtt->base.start; 1449 uint64_t length = ppgtt->base.total; 1450 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 1451 I915_CACHE_LLC, true); 1452 1453 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 1454 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); 1455 } else { 1456 uint64_t pml4e; 1457 struct i915_pml4 *pml4 = &ppgtt->pml4; 1458 struct i915_page_directory_pointer *pdp; 1459 1460 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1461 if (!test_bit(pml4e, pml4->used_pml4es)) 1462 continue; 1463 1464 seq_printf(m, " PML4E #%llu\n", pml4e); 1465 gen8_dump_pdp(pdp, start, length, scratch_pte, m); 1466 } 1467 } 1468 } 1469 1470 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) 1471 { 1472 unsigned long *new_page_dirs, *new_page_tables; 1473 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1474 int ret; 1475 1476 /* We allocate temp bitmap for page tables for no gain 1477 * but as this is for init only, lets keep the things simple 1478 */ 1479 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1480 if (ret) 1481 return ret; 1482 1483 /* Allocate for all pdps regardless of how the ppgtt 1484 * was defined. 1485 */ 1486 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, 1487 0, 1ULL << 32, 1488 new_page_dirs); 1489 if (!ret) 1490 *ppgtt->pdp.used_pdpes = *new_page_dirs; 1491 1492 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1493 1494 return ret; 1495 } 1496 1497 /* 1498 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1499 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1500 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1501 * space. 1502 * 1503 */ 1504 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1505 { 1506 int ret; 1507 1508 ret = gen8_init_scratch(&ppgtt->base); 1509 if (ret) 1510 return ret; 1511 1512 ppgtt->base.start = 0; 1513 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1514 ppgtt->base.allocate_va_range = gen8_alloc_va_range; 1515 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; 1516 ppgtt->base.clear_range = gen8_ppgtt_clear_range; 1517 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1518 ppgtt->base.bind_vma = ppgtt_bind_vma; 1519 ppgtt->debug_dump = gen8_dump_ppgtt; 1520 1521 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { 1522 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); 1523 if (ret) 1524 goto free_scratch; 1525 1526 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); 1527 1528 ppgtt->base.total = 1ULL << 48; 1529 ppgtt->switch_mm = gen8_48b_mm_switch; 1530 } else { 1531 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); 1532 if (ret) 1533 goto free_scratch; 1534 1535 ppgtt->base.total = 1ULL << 32; 1536 ppgtt->switch_mm = gen8_legacy_mm_switch; 1537 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, 1538 0, 0, 1539 GEN8_PML4E_SHIFT); 1540 1541 if (intel_vgpu_active(ppgtt->base.dev)) { 1542 ret = gen8_preallocate_top_level_pdps(ppgtt); 1543 if (ret) 1544 goto free_scratch; 1545 } 1546 } 1547 1548 if (intel_vgpu_active(ppgtt->base.dev)) 1549 gen8_ppgtt_notify_vgt(ppgtt, true); 1550 1551 return 0; 1552 1553 free_scratch: 1554 gen8_free_scratch(&ppgtt->base); 1555 return ret; 1556 } 1557 1558 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1559 { 1560 struct i915_address_space *vm = &ppgtt->base; 1561 struct i915_page_table *unused; 1562 gen6_pte_t scratch_pte; 1563 uint32_t pd_entry; 1564 uint32_t pte, pde, temp; 1565 uint32_t start = ppgtt->base.start, length = ppgtt->base.total; 1566 1567 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1568 I915_CACHE_LLC, true, 0); 1569 1570 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { 1571 u32 expected; 1572 gen6_pte_t *pt_vaddr; 1573 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1574 pd_entry = readl(ppgtt->pd_addr + pde); 1575 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1576 1577 if (pd_entry != expected) 1578 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1579 pde, 1580 pd_entry, 1581 expected); 1582 seq_printf(m, "\tPDE: %x\n", pd_entry); 1583 1584 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); 1585 1586 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1587 unsigned long va = 1588 (pde * PAGE_SIZE * GEN6_PTES) + 1589 (pte * PAGE_SIZE); 1590 int i; 1591 bool found = false; 1592 for (i = 0; i < 4; i++) 1593 if (pt_vaddr[pte + i] != scratch_pte) 1594 found = true; 1595 if (!found) 1596 continue; 1597 1598 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1599 for (i = 0; i < 4; i++) { 1600 if (pt_vaddr[pte + i] != scratch_pte) 1601 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1602 else 1603 seq_puts(m, " SCRATCH "); 1604 } 1605 seq_puts(m, "\n"); 1606 } 1607 kunmap_px(ppgtt, pt_vaddr); 1608 } 1609 } 1610 1611 /* Write pde (index) from the page directory @pd to the page table @pt */ 1612 static void gen6_write_pde(struct i915_page_directory *pd, 1613 const int pde, struct i915_page_table *pt) 1614 { 1615 /* Caller needs to make sure the write completes if necessary */ 1616 struct i915_hw_ppgtt *ppgtt = 1617 container_of(pd, struct i915_hw_ppgtt, pd); 1618 u32 pd_entry; 1619 1620 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); 1621 pd_entry |= GEN6_PDE_VALID; 1622 1623 writel(pd_entry, ppgtt->pd_addr + pde); 1624 } 1625 1626 /* Write all the page tables found in the ppgtt structure to incrementing page 1627 * directories. */ 1628 static void gen6_write_page_range(struct drm_i915_private *dev_priv, 1629 struct i915_page_directory *pd, 1630 uint32_t start, uint32_t length) 1631 { 1632 struct i915_page_table *pt; 1633 uint32_t pde, temp; 1634 1635 gen6_for_each_pde(pt, pd, start, length, temp, pde) 1636 gen6_write_pde(pd, pde, pt); 1637 1638 /* Make sure write is complete before other code can use this page 1639 * table. Also require for WC mapped PTEs */ 1640 readl(dev_priv->gtt.gsm); 1641 } 1642 1643 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1644 { 1645 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1646 1647 return (ppgtt->pd.base.ggtt_offset / 64) << 16; 1648 } 1649 1650 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1651 struct drm_i915_gem_request *req) 1652 { 1653 struct intel_engine_cs *ring = req->ring; 1654 int ret; 1655 1656 /* NB: TLBs must be flushed and invalidated before a switch */ 1657 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1658 if (ret) 1659 return ret; 1660 1661 ret = intel_ring_begin(req, 6); 1662 if (ret) 1663 return ret; 1664 1665 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1666 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); 1667 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1668 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); 1669 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1670 intel_ring_emit(ring, MI_NOOP); 1671 intel_ring_advance(ring); 1672 1673 return 0; 1674 } 1675 1676 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, 1677 struct drm_i915_gem_request *req) 1678 { 1679 struct intel_engine_cs *ring = req->ring; 1680 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); 1681 1682 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1683 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1684 return 0; 1685 } 1686 1687 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1688 struct drm_i915_gem_request *req) 1689 { 1690 struct intel_engine_cs *ring = req->ring; 1691 int ret; 1692 1693 /* NB: TLBs must be flushed and invalidated before a switch */ 1694 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1695 if (ret) 1696 return ret; 1697 1698 ret = intel_ring_begin(req, 6); 1699 if (ret) 1700 return ret; 1701 1702 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1703 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); 1704 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1705 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); 1706 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1707 intel_ring_emit(ring, MI_NOOP); 1708 intel_ring_advance(ring); 1709 1710 /* XXX: RCS is the only one to auto invalidate the TLBs? */ 1711 if (ring->id != RCS) { 1712 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1713 if (ret) 1714 return ret; 1715 } 1716 1717 return 0; 1718 } 1719 1720 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1721 struct drm_i915_gem_request *req) 1722 { 1723 struct intel_engine_cs *ring = req->ring; 1724 struct drm_device *dev = ppgtt->base.dev; 1725 struct drm_i915_private *dev_priv = dev->dev_private; 1726 1727 1728 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1729 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1730 1731 POSTING_READ(RING_PP_DIR_DCLV(ring)); 1732 1733 return 0; 1734 } 1735 1736 static void gen8_ppgtt_enable(struct drm_device *dev) 1737 { 1738 struct drm_i915_private *dev_priv = dev->dev_private; 1739 struct intel_engine_cs *ring; 1740 int j; 1741 1742 for_each_ring(ring, dev_priv, j) { 1743 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; 1744 I915_WRITE(RING_MODE_GEN7(ring), 1745 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1746 } 1747 } 1748 1749 static void gen7_ppgtt_enable(struct drm_device *dev) 1750 { 1751 struct drm_i915_private *dev_priv = dev->dev_private; 1752 struct intel_engine_cs *ring; 1753 uint32_t ecochk, ecobits; 1754 int i; 1755 1756 ecobits = I915_READ(GAC_ECO_BITS); 1757 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1758 1759 ecochk = I915_READ(GAM_ECOCHK); 1760 if (IS_HASWELL(dev)) { 1761 ecochk |= ECOCHK_PPGTT_WB_HSW; 1762 } else { 1763 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1764 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1765 } 1766 I915_WRITE(GAM_ECOCHK, ecochk); 1767 1768 for_each_ring(ring, dev_priv, i) { 1769 /* GFX_MODE is per-ring on gen7+ */ 1770 I915_WRITE(RING_MODE_GEN7(ring), 1771 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1772 } 1773 } 1774 1775 static void gen6_ppgtt_enable(struct drm_device *dev) 1776 { 1777 struct drm_i915_private *dev_priv = dev->dev_private; 1778 uint32_t ecochk, gab_ctl, ecobits; 1779 1780 ecobits = I915_READ(GAC_ECO_BITS); 1781 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1782 ECOBITS_PPGTT_CACHE64B); 1783 1784 gab_ctl = I915_READ(GAB_CTL); 1785 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1786 1787 ecochk = I915_READ(GAM_ECOCHK); 1788 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1789 1790 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1791 } 1792 1793 /* PPGTT support for Sandybdrige/Gen6 and later */ 1794 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1795 uint64_t start, 1796 uint64_t length, 1797 bool use_scratch) 1798 { 1799 struct i915_hw_ppgtt *ppgtt = 1800 container_of(vm, struct i915_hw_ppgtt, base); 1801 gen6_pte_t *pt_vaddr, scratch_pte; 1802 unsigned first_entry = start >> PAGE_SHIFT; 1803 unsigned num_entries = length >> PAGE_SHIFT; 1804 unsigned act_pt = first_entry / GEN6_PTES; 1805 unsigned first_pte = first_entry % GEN6_PTES; 1806 unsigned last_pte, i; 1807 1808 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1809 I915_CACHE_LLC, true, 0); 1810 1811 while (num_entries) { 1812 last_pte = first_pte + num_entries; 1813 if (last_pte > GEN6_PTES) 1814 last_pte = GEN6_PTES; 1815 1816 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1817 1818 for (i = first_pte; i < last_pte; i++) 1819 pt_vaddr[i] = scratch_pte; 1820 1821 kunmap_px(ppgtt, pt_vaddr); 1822 1823 num_entries -= last_pte - first_pte; 1824 first_pte = 0; 1825 act_pt++; 1826 } 1827 } 1828 1829 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 1830 struct sg_table *pages, 1831 uint64_t start, 1832 enum i915_cache_level cache_level, u32 flags) 1833 { 1834 struct i915_hw_ppgtt *ppgtt = 1835 container_of(vm, struct i915_hw_ppgtt, base); 1836 gen6_pte_t *pt_vaddr; 1837 unsigned first_entry = start >> PAGE_SHIFT; 1838 unsigned act_pt = first_entry / GEN6_PTES; 1839 unsigned act_pte = first_entry % GEN6_PTES; 1840 struct sg_page_iter sg_iter; 1841 1842 pt_vaddr = NULL; 1843 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 1844 if (pt_vaddr == NULL) 1845 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1846 1847 pt_vaddr[act_pte] = 1848 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), 1849 cache_level, true, flags); 1850 1851 if (++act_pte == GEN6_PTES) { 1852 kunmap_px(ppgtt, pt_vaddr); 1853 pt_vaddr = NULL; 1854 act_pt++; 1855 act_pte = 0; 1856 } 1857 } 1858 if (pt_vaddr) 1859 kunmap_px(ppgtt, pt_vaddr); 1860 } 1861 1862 static int gen6_alloc_va_range(struct i915_address_space *vm, 1863 uint64_t start_in, uint64_t length_in) 1864 { 1865 DECLARE_BITMAP(new_page_tables, I915_PDES); 1866 struct drm_device *dev = vm->dev; 1867 struct drm_i915_private *dev_priv = dev->dev_private; 1868 struct i915_hw_ppgtt *ppgtt = 1869 container_of(vm, struct i915_hw_ppgtt, base); 1870 struct i915_page_table *pt; 1871 uint32_t start, length, start_save, length_save; 1872 uint32_t pde, temp; 1873 int ret; 1874 1875 if (WARN_ON(start_in + length_in > ppgtt->base.total)) 1876 return -ENODEV; 1877 1878 start = start_save = start_in; 1879 length = length_save = length_in; 1880 1881 bitmap_zero(new_page_tables, I915_PDES); 1882 1883 /* The allocation is done in two stages so that we can bail out with 1884 * minimal amount of pain. The first stage finds new page tables that 1885 * need allocation. The second stage marks use ptes within the page 1886 * tables. 1887 */ 1888 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1889 if (pt != vm->scratch_pt) { 1890 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); 1891 continue; 1892 } 1893 1894 /* We've already allocated a page table */ 1895 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 1896 1897 pt = alloc_pt(dev); 1898 if (IS_ERR(pt)) { 1899 ret = PTR_ERR(pt); 1900 goto unwind_out; 1901 } 1902 1903 gen6_initialize_pt(vm, pt); 1904 1905 ppgtt->pd.page_table[pde] = pt; 1906 __set_bit(pde, new_page_tables); 1907 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); 1908 } 1909 1910 start = start_save; 1911 length = length_save; 1912 1913 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1914 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); 1915 1916 bitmap_zero(tmp_bitmap, GEN6_PTES); 1917 bitmap_set(tmp_bitmap, gen6_pte_index(start), 1918 gen6_pte_count(start, length)); 1919 1920 if (__test_and_clear_bit(pde, new_page_tables)) 1921 gen6_write_pde(&ppgtt->pd, pde, pt); 1922 1923 trace_i915_page_table_entry_map(vm, pde, pt, 1924 gen6_pte_index(start), 1925 gen6_pte_count(start, length), 1926 GEN6_PTES); 1927 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, 1928 GEN6_PTES); 1929 } 1930 1931 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); 1932 1933 /* Make sure write is complete before other code can use this page 1934 * table. Also require for WC mapped PTEs */ 1935 readl(dev_priv->gtt.gsm); 1936 1937 mark_tlbs_dirty(ppgtt); 1938 return 0; 1939 1940 unwind_out: 1941 for_each_set_bit(pde, new_page_tables, I915_PDES) { 1942 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 1943 1944 ppgtt->pd.page_table[pde] = vm->scratch_pt; 1945 free_pt(vm->dev, pt); 1946 } 1947 1948 mark_tlbs_dirty(ppgtt); 1949 return ret; 1950 } 1951 1952 static int gen6_init_scratch(struct i915_address_space *vm) 1953 { 1954 struct drm_device *dev = vm->dev; 1955 1956 vm->scratch_page = alloc_scratch_page(dev); 1957 if (IS_ERR(vm->scratch_page)) 1958 return PTR_ERR(vm->scratch_page); 1959 1960 vm->scratch_pt = alloc_pt(dev); 1961 if (IS_ERR(vm->scratch_pt)) { 1962 free_scratch_page(dev, vm->scratch_page); 1963 return PTR_ERR(vm->scratch_pt); 1964 } 1965 1966 gen6_initialize_pt(vm, vm->scratch_pt); 1967 1968 return 0; 1969 } 1970 1971 static void gen6_free_scratch(struct i915_address_space *vm) 1972 { 1973 struct drm_device *dev = vm->dev; 1974 1975 free_pt(dev, vm->scratch_pt); 1976 free_scratch_page(dev, vm->scratch_page); 1977 } 1978 1979 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 1980 { 1981 struct i915_hw_ppgtt *ppgtt = 1982 container_of(vm, struct i915_hw_ppgtt, base); 1983 struct i915_page_table *pt; 1984 uint32_t pde; 1985 1986 drm_mm_remove_node(&ppgtt->node); 1987 1988 gen6_for_all_pdes(pt, ppgtt, pde) { 1989 if (pt != vm->scratch_pt) 1990 free_pt(ppgtt->base.dev, pt); 1991 } 1992 1993 gen6_free_scratch(vm); 1994 } 1995 1996 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 1997 { 1998 struct i915_address_space *vm = &ppgtt->base; 1999 struct drm_device *dev = ppgtt->base.dev; 2000 struct drm_i915_private *dev_priv = dev->dev_private; 2001 bool retried = false; 2002 int ret; 2003 2004 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 2005 * allocator works in address space sizes, so it's multiplied by page 2006 * size. We allocate at the top of the GTT to avoid fragmentation. 2007 */ 2008 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); 2009 2010 ret = gen6_init_scratch(vm); 2011 if (ret) 2012 return ret; 2013 2014 alloc: 2015 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, 2016 &ppgtt->node, GEN6_PD_SIZE, 2017 GEN6_PD_ALIGN, 0, 2018 0, dev_priv->gtt.base.total, 2019 DRM_MM_TOPDOWN); 2020 if (ret == -ENOSPC && !retried) { 2021 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, 2022 GEN6_PD_SIZE, GEN6_PD_ALIGN, 2023 I915_CACHE_NONE, 2024 0, dev_priv->gtt.base.total, 2025 0); 2026 if (ret) 2027 goto err_out; 2028 2029 retried = true; 2030 goto alloc; 2031 } 2032 2033 if (ret) 2034 goto err_out; 2035 2036 2037 if (ppgtt->node.start < dev_priv->gtt.mappable_end) 2038 DRM_DEBUG("Forced to use aperture for PDEs\n"); 2039 2040 return 0; 2041 2042 err_out: 2043 gen6_free_scratch(vm); 2044 return ret; 2045 } 2046 2047 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 2048 { 2049 return gen6_ppgtt_allocate_page_directories(ppgtt); 2050 } 2051 2052 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 2053 uint64_t start, uint64_t length) 2054 { 2055 struct i915_page_table *unused; 2056 uint32_t pde, temp; 2057 2058 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) 2059 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 2060 } 2061 2062 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 2063 { 2064 struct drm_device *dev = ppgtt->base.dev; 2065 struct drm_i915_private *dev_priv = dev->dev_private; 2066 int ret; 2067 2068 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; 2069 if (IS_GEN6(dev)) { 2070 ppgtt->switch_mm = gen6_mm_switch; 2071 } else if (IS_HASWELL(dev)) { 2072 ppgtt->switch_mm = hsw_mm_switch; 2073 } else if (IS_GEN7(dev)) { 2074 ppgtt->switch_mm = gen7_mm_switch; 2075 } else 2076 BUG(); 2077 2078 if (intel_vgpu_active(dev)) 2079 ppgtt->switch_mm = vgpu_mm_switch; 2080 2081 ret = gen6_ppgtt_alloc(ppgtt); 2082 if (ret) 2083 return ret; 2084 2085 ppgtt->base.allocate_va_range = gen6_alloc_va_range; 2086 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 2087 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 2088 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 2089 ppgtt->base.bind_vma = ppgtt_bind_vma; 2090 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 2091 ppgtt->base.start = 0; 2092 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 2093 ppgtt->debug_dump = gen6_dump_ppgtt; 2094 2095 ppgtt->pd.base.ggtt_offset = 2096 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 2097 2098 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + 2099 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 2100 2101 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 2102 2103 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); 2104 2105 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", 2106 ppgtt->node.size >> 20, 2107 ppgtt->node.start / PAGE_SIZE); 2108 2109 DRM_DEBUG("Adding PPGTT at offset %x\n", 2110 ppgtt->pd.base.ggtt_offset << 10); 2111 2112 return 0; 2113 } 2114 2115 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2116 { 2117 ppgtt->base.dev = dev; 2118 2119 if (INTEL_INFO(dev)->gen < 8) 2120 return gen6_ppgtt_init(ppgtt); 2121 else 2122 return gen8_ppgtt_init(ppgtt); 2123 } 2124 2125 static void i915_address_space_init(struct i915_address_space *vm, 2126 struct drm_i915_private *dev_priv) 2127 { 2128 drm_mm_init(&vm->mm, vm->start, vm->total); 2129 vm->dev = dev_priv->dev; 2130 INIT_LIST_HEAD(&vm->active_list); 2131 INIT_LIST_HEAD(&vm->inactive_list); 2132 list_add_tail(&vm->global_link, &dev_priv->vm_list); 2133 } 2134 2135 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2136 { 2137 struct drm_i915_private *dev_priv = dev->dev_private; 2138 int ret = 0; 2139 2140 ret = __hw_ppgtt_init(dev, ppgtt); 2141 if (ret == 0) { 2142 kref_init(&ppgtt->ref); 2143 i915_address_space_init(&ppgtt->base, dev_priv); 2144 } 2145 2146 return ret; 2147 } 2148 2149 int i915_ppgtt_init_hw(struct drm_device *dev) 2150 { 2151 /* In the case of execlists, PPGTT is enabled by the context descriptor 2152 * and the PDPs are contained within the context itself. We don't 2153 * need to do anything here. */ 2154 if (i915.enable_execlists) 2155 return 0; 2156 2157 if (!USES_PPGTT(dev)) 2158 return 0; 2159 2160 if (IS_GEN6(dev)) 2161 gen6_ppgtt_enable(dev); 2162 else if (IS_GEN7(dev)) 2163 gen7_ppgtt_enable(dev); 2164 else if (INTEL_INFO(dev)->gen >= 8) 2165 gen8_ppgtt_enable(dev); 2166 else 2167 MISSING_CASE(INTEL_INFO(dev)->gen); 2168 2169 return 0; 2170 } 2171 2172 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) 2173 { 2174 struct drm_i915_private *dev_priv = req->ring->dev->dev_private; 2175 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2176 2177 if (i915.enable_execlists) 2178 return 0; 2179 2180 if (!ppgtt) 2181 return 0; 2182 2183 return ppgtt->switch_mm(ppgtt, req); 2184 } 2185 2186 struct i915_hw_ppgtt * 2187 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) 2188 { 2189 struct i915_hw_ppgtt *ppgtt; 2190 int ret; 2191 2192 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2193 if (!ppgtt) 2194 return ERR_PTR(-ENOMEM); 2195 2196 ret = i915_ppgtt_init(dev, ppgtt); 2197 if (ret) { 2198 kfree(ppgtt); 2199 return ERR_PTR(ret); 2200 } 2201 2202 ppgtt->file_priv = fpriv; 2203 2204 trace_i915_ppgtt_create(&ppgtt->base); 2205 2206 return ppgtt; 2207 } 2208 2209 void i915_ppgtt_release(struct kref *kref) 2210 { 2211 struct i915_hw_ppgtt *ppgtt = 2212 container_of(kref, struct i915_hw_ppgtt, ref); 2213 2214 trace_i915_ppgtt_release(&ppgtt->base); 2215 2216 /* vmas should already be unbound */ 2217 WARN_ON(!list_empty(&ppgtt->base.active_list)); 2218 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 2219 2220 list_del(&ppgtt->base.global_link); 2221 drm_mm_takedown(&ppgtt->base.mm); 2222 2223 ppgtt->base.cleanup(&ppgtt->base); 2224 kfree(ppgtt); 2225 } 2226 2227 extern int intel_iommu_gfx_mapped; 2228 /* Certain Gen5 chipsets require require idling the GPU before 2229 * unmapping anything from the GTT when VT-d is enabled. 2230 */ 2231 static bool needs_idle_maps(struct drm_device *dev) 2232 { 2233 #ifdef CONFIG_INTEL_IOMMU 2234 /* Query intel_iommu to see if we need the workaround. Presumably that 2235 * was loaded first. 2236 */ 2237 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 2238 return true; 2239 #endif 2240 return false; 2241 } 2242 2243 static bool do_idling(struct drm_i915_private *dev_priv) 2244 { 2245 bool ret = dev_priv->mm.interruptible; 2246 2247 if (unlikely(dev_priv->gtt.do_idle_maps)) { 2248 dev_priv->mm.interruptible = false; 2249 if (i915_gpu_idle(dev_priv->dev)) { 2250 DRM_ERROR("Couldn't idle GPU\n"); 2251 /* Wait a bit, in hopes it avoids the hang */ 2252 udelay(10); 2253 } 2254 } 2255 2256 return ret; 2257 } 2258 2259 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 2260 { 2261 if (unlikely(dev_priv->gtt.do_idle_maps)) 2262 dev_priv->mm.interruptible = interruptible; 2263 } 2264 2265 void i915_check_and_clear_faults(struct drm_device *dev) 2266 { 2267 struct drm_i915_private *dev_priv = dev->dev_private; 2268 struct intel_engine_cs *ring; 2269 int i; 2270 2271 if (INTEL_INFO(dev)->gen < 6) 2272 return; 2273 2274 for_each_ring(ring, dev_priv, i) { 2275 u32 fault_reg; 2276 fault_reg = I915_READ(RING_FAULT_REG(ring)); 2277 if (fault_reg & RING_FAULT_VALID) { 2278 DRM_DEBUG_DRIVER("Unexpected fault\n" 2279 "\tAddr: 0x%08lx\n" 2280 "\tAddress space: %s\n" 2281 "\tSource ID: %d\n" 2282 "\tType: %d\n", 2283 fault_reg & PAGE_MASK, 2284 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 2285 RING_FAULT_SRCID(fault_reg), 2286 RING_FAULT_FAULT_TYPE(fault_reg)); 2287 I915_WRITE(RING_FAULT_REG(ring), 2288 fault_reg & ~RING_FAULT_VALID); 2289 } 2290 } 2291 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); 2292 } 2293 2294 static void i915_ggtt_flush(struct drm_i915_private *dev_priv) 2295 { 2296 if (INTEL_INFO(dev_priv->dev)->gen < 6) { 2297 intel_gtt_chipset_flush(); 2298 } else { 2299 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2300 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2301 } 2302 } 2303 2304 void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 2305 { 2306 struct drm_i915_private *dev_priv = dev->dev_private; 2307 2308 /* Don't bother messing with faults pre GEN6 as we have little 2309 * documentation supporting that it's a good idea. 2310 */ 2311 if (INTEL_INFO(dev)->gen < 6) 2312 return; 2313 2314 i915_check_and_clear_faults(dev); 2315 2316 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 2317 dev_priv->gtt.base.start, 2318 dev_priv->gtt.base.total, 2319 true); 2320 2321 i915_ggtt_flush(dev_priv); 2322 } 2323 2324 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 2325 { 2326 if (!dma_map_sg(&obj->base.dev->pdev->dev, 2327 obj->pages->sgl, obj->pages->nents, 2328 PCI_DMA_BIDIRECTIONAL)) 2329 return -ENOSPC; 2330 2331 return 0; 2332 } 2333 2334 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2335 { 2336 #ifdef writeq 2337 writeq(pte, addr); 2338 #else 2339 iowrite32((u32)pte, addr); 2340 iowrite32(pte >> 32, addr + 4); 2341 #endif 2342 } 2343 2344 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2345 struct sg_table *st, 2346 uint64_t start, 2347 enum i915_cache_level level, u32 unused) 2348 { 2349 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2350 unsigned first_entry = start >> PAGE_SHIFT; 2351 gen8_pte_t __iomem *gtt_entries = 2352 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2353 int i = 0; 2354 struct sg_page_iter sg_iter; 2355 dma_addr_t addr = 0; /* shut up gcc */ 2356 int rpm_atomic_seq; 2357 2358 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2359 2360 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2361 addr = sg_dma_address(sg_iter.sg) + 2362 (sg_iter.sg_pgoffset << PAGE_SHIFT); 2363 gen8_set_pte(>t_entries[i], 2364 gen8_pte_encode(addr, level, true)); 2365 i++; 2366 } 2367 2368 /* 2369 * XXX: This serves as a posting read to make sure that the PTE has 2370 * actually been updated. There is some concern that even though 2371 * registers and PTEs are within the same BAR that they are potentially 2372 * of NUMA access patterns. Therefore, even with the way we assume 2373 * hardware should work, we must keep this posting read for paranoia. 2374 */ 2375 if (i != 0) 2376 WARN_ON(readq(>t_entries[i-1]) 2377 != gen8_pte_encode(addr, level, true)); 2378 2379 /* This next bit makes the above posting read even more important. We 2380 * want to flush the TLBs only after we're certain all the PTE updates 2381 * have finished. 2382 */ 2383 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2384 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2385 2386 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2387 } 2388 2389 struct insert_entries { 2390 struct i915_address_space *vm; 2391 struct sg_table *st; 2392 uint64_t start; 2393 enum i915_cache_level level; 2394 u32 flags; 2395 }; 2396 2397 static int gen8_ggtt_insert_entries__cb(void *_arg) 2398 { 2399 struct insert_entries *arg = _arg; 2400 gen8_ggtt_insert_entries(arg->vm, arg->st, 2401 arg->start, arg->level, arg->flags); 2402 return 0; 2403 } 2404 2405 static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm, 2406 struct sg_table *st, 2407 uint64_t start, 2408 enum i915_cache_level level, 2409 u32 flags) 2410 { 2411 struct insert_entries arg = { vm, st, start, level, flags }; 2412 stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL); 2413 } 2414 2415 /* 2416 * Binds an object into the global gtt with the specified cache level. The object 2417 * will be accessible to the GPU via commands whose operands reference offsets 2418 * within the global GTT as well as accessible by the GPU through the GMADR 2419 * mapped BAR (dev_priv->mm.gtt->gtt). 2420 */ 2421 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 2422 struct sg_table *st, 2423 uint64_t start, 2424 enum i915_cache_level level, u32 flags) 2425 { 2426 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2427 unsigned first_entry = start >> PAGE_SHIFT; 2428 gen6_pte_t __iomem *gtt_entries = 2429 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2430 int i = 0; 2431 struct sg_page_iter sg_iter; 2432 dma_addr_t addr = 0; 2433 int rpm_atomic_seq; 2434 2435 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2436 2437 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2438 addr = sg_page_iter_dma_address(&sg_iter); 2439 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); 2440 i++; 2441 } 2442 2443 /* XXX: This serves as a posting read to make sure that the PTE has 2444 * actually been updated. There is some concern that even though 2445 * registers and PTEs are within the same BAR that they are potentially 2446 * of NUMA access patterns. Therefore, even with the way we assume 2447 * hardware should work, we must keep this posting read for paranoia. 2448 */ 2449 if (i != 0) { 2450 unsigned long gtt = readl(>t_entries[i-1]); 2451 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); 2452 } 2453 2454 /* This next bit makes the above posting read even more important. We 2455 * want to flush the TLBs only after we're certain all the PTE updates 2456 * have finished. 2457 */ 2458 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2459 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2460 2461 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2462 } 2463 2464 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2465 uint64_t start, 2466 uint64_t length, 2467 bool use_scratch) 2468 { 2469 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2470 unsigned first_entry = start >> PAGE_SHIFT; 2471 unsigned num_entries = length >> PAGE_SHIFT; 2472 gen8_pte_t scratch_pte, __iomem *gtt_base = 2473 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2474 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2475 int i; 2476 int rpm_atomic_seq; 2477 2478 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2479 2480 if (WARN(num_entries > max_entries, 2481 "First entry = %d; Num entries = %d (max=%d)\n", 2482 first_entry, num_entries, max_entries)) 2483 num_entries = max_entries; 2484 2485 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 2486 I915_CACHE_LLC, 2487 use_scratch); 2488 for (i = 0; i < num_entries; i++) 2489 gen8_set_pte(>t_base[i], scratch_pte); 2490 readl(gtt_base); 2491 2492 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2493 } 2494 2495 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2496 uint64_t start, 2497 uint64_t length, 2498 bool use_scratch) 2499 { 2500 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2501 unsigned first_entry = start >> PAGE_SHIFT; 2502 unsigned num_entries = length >> PAGE_SHIFT; 2503 gen6_pte_t scratch_pte, __iomem *gtt_base = 2504 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2505 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2506 int i; 2507 int rpm_atomic_seq; 2508 2509 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2510 2511 if (WARN(num_entries > max_entries, 2512 "First entry = %d; Num entries = %d (max=%d)\n", 2513 first_entry, num_entries, max_entries)) 2514 num_entries = max_entries; 2515 2516 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 2517 I915_CACHE_LLC, use_scratch, 0); 2518 2519 for (i = 0; i < num_entries; i++) 2520 iowrite32(scratch_pte, >t_base[i]); 2521 readl(gtt_base); 2522 2523 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2524 } 2525 2526 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2527 struct sg_table *pages, 2528 uint64_t start, 2529 enum i915_cache_level cache_level, u32 unused) 2530 { 2531 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2532 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2533 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2534 int rpm_atomic_seq; 2535 2536 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2537 2538 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2539 2540 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2541 2542 } 2543 2544 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2545 uint64_t start, 2546 uint64_t length, 2547 bool unused) 2548 { 2549 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2550 unsigned first_entry = start >> PAGE_SHIFT; 2551 unsigned num_entries = length >> PAGE_SHIFT; 2552 int rpm_atomic_seq; 2553 2554 rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); 2555 2556 intel_gtt_clear_range(first_entry, num_entries); 2557 2558 assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); 2559 } 2560 2561 static int ggtt_bind_vma(struct i915_vma *vma, 2562 enum i915_cache_level cache_level, 2563 u32 flags) 2564 { 2565 struct drm_i915_gem_object *obj = vma->obj; 2566 u32 pte_flags = 0; 2567 int ret; 2568 2569 ret = i915_get_ggtt_vma_pages(vma); 2570 if (ret) 2571 return ret; 2572 2573 /* Currently applicable only to VLV */ 2574 if (obj->gt_ro) 2575 pte_flags |= PTE_READ_ONLY; 2576 2577 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, 2578 vma->node.start, 2579 cache_level, pte_flags); 2580 2581 /* 2582 * Without aliasing PPGTT there's no difference between 2583 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 2584 * upgrade to both bound if we bind either to avoid double-binding. 2585 */ 2586 vma->bound |= GLOBAL_BIND | LOCAL_BIND; 2587 2588 return 0; 2589 } 2590 2591 static int aliasing_gtt_bind_vma(struct i915_vma *vma, 2592 enum i915_cache_level cache_level, 2593 u32 flags) 2594 { 2595 struct drm_device *dev = vma->vm->dev; 2596 struct drm_i915_private *dev_priv = dev->dev_private; 2597 struct drm_i915_gem_object *obj = vma->obj; 2598 struct sg_table *pages = obj->pages; 2599 u32 pte_flags = 0; 2600 int ret; 2601 2602 ret = i915_get_ggtt_vma_pages(vma); 2603 if (ret) 2604 return ret; 2605 pages = vma->ggtt_view.pages; 2606 2607 /* Currently applicable only to VLV */ 2608 if (obj->gt_ro) 2609 pte_flags |= PTE_READ_ONLY; 2610 2611 2612 if (flags & GLOBAL_BIND) { 2613 vma->vm->insert_entries(vma->vm, pages, 2614 vma->node.start, 2615 cache_level, pte_flags); 2616 } 2617 2618 if (flags & LOCAL_BIND) { 2619 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2620 appgtt->base.insert_entries(&appgtt->base, pages, 2621 vma->node.start, 2622 cache_level, pte_flags); 2623 } 2624 2625 return 0; 2626 } 2627 2628 static void ggtt_unbind_vma(struct i915_vma *vma) 2629 { 2630 struct drm_device *dev = vma->vm->dev; 2631 struct drm_i915_private *dev_priv = dev->dev_private; 2632 struct drm_i915_gem_object *obj = vma->obj; 2633 const uint64_t size = min_t(uint64_t, 2634 obj->base.size, 2635 vma->node.size); 2636 2637 if (vma->bound & GLOBAL_BIND) { 2638 vma->vm->clear_range(vma->vm, 2639 vma->node.start, 2640 size, 2641 true); 2642 } 2643 2644 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { 2645 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2646 2647 appgtt->base.clear_range(&appgtt->base, 2648 vma->node.start, 2649 size, 2650 true); 2651 } 2652 } 2653 2654 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 2655 { 2656 struct drm_device *dev = obj->base.dev; 2657 struct drm_i915_private *dev_priv = dev->dev_private; 2658 bool interruptible; 2659 2660 interruptible = do_idling(dev_priv); 2661 2662 dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents, 2663 PCI_DMA_BIDIRECTIONAL); 2664 2665 undo_idling(dev_priv, interruptible); 2666 } 2667 2668 static void i915_gtt_color_adjust(struct drm_mm_node *node, 2669 unsigned long color, 2670 u64 *start, 2671 u64 *end) 2672 { 2673 if (node->color != color) 2674 *start += 4096; 2675 2676 if (!list_empty(&node->node_list)) { 2677 node = list_entry(node->node_list.next, 2678 struct drm_mm_node, 2679 node_list); 2680 if (node->allocated && node->color != color) 2681 *end -= 4096; 2682 } 2683 } 2684 2685 static int i915_gem_setup_global_gtt(struct drm_device *dev, 2686 u64 start, 2687 u64 mappable_end, 2688 u64 end) 2689 { 2690 /* Let GEM Manage all of the aperture. 2691 * 2692 * However, leave one page at the end still bound to the scratch page. 2693 * There are a number of places where the hardware apparently prefetches 2694 * past the end of the object, and we've seen multiple hangs with the 2695 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2696 * aperture. One page should be enough to keep any prefetching inside 2697 * of the aperture. 2698 */ 2699 struct drm_i915_private *dev_priv = dev->dev_private; 2700 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2701 struct drm_mm_node *entry; 2702 struct drm_i915_gem_object *obj; 2703 unsigned long hole_start, hole_end; 2704 int ret; 2705 2706 BUG_ON(mappable_end > end); 2707 2708 ggtt_vm->start = start; 2709 2710 /* Subtract the guard page before address space initialization to 2711 * shrink the range used by drm_mm */ 2712 ggtt_vm->total = end - start - PAGE_SIZE; 2713 i915_address_space_init(ggtt_vm, dev_priv); 2714 ggtt_vm->total += PAGE_SIZE; 2715 2716 if (intel_vgpu_active(dev)) { 2717 ret = intel_vgt_balloon(dev); 2718 if (ret) 2719 return ret; 2720 } 2721 2722 if (!HAS_LLC(dev)) 2723 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; 2724 2725 /* Mark any preallocated objects as occupied */ 2726 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 2727 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 2728 2729 DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n", 2730 i915_gem_obj_ggtt_offset(obj), obj->base.size); 2731 2732 WARN_ON(i915_gem_obj_ggtt_bound(obj)); 2733 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); 2734 if (ret) { 2735 DRM_DEBUG_KMS("Reservation failed: %i\n", ret); 2736 return ret; 2737 } 2738 vma->bound |= GLOBAL_BIND; 2739 __i915_vma_set_map_and_fenceable(vma); 2740 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); 2741 } 2742 2743 /* Clear any non-preallocated blocks */ 2744 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { 2745 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 2746 hole_start, hole_end); 2747 ggtt_vm->clear_range(ggtt_vm, hole_start, 2748 hole_end - hole_start, true); 2749 } 2750 2751 /* And finally clear the reserved guard page */ 2752 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); 2753 2754 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { 2755 struct i915_hw_ppgtt *ppgtt; 2756 2757 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2758 if (!ppgtt) 2759 return -ENOMEM; 2760 2761 ret = __hw_ppgtt_init(dev, ppgtt); 2762 if (ret) { 2763 ppgtt->base.cleanup(&ppgtt->base); 2764 kfree(ppgtt); 2765 return ret; 2766 } 2767 2768 if (ppgtt->base.allocate_va_range) 2769 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, 2770 ppgtt->base.total); 2771 if (ret) { 2772 ppgtt->base.cleanup(&ppgtt->base); 2773 kfree(ppgtt); 2774 return ret; 2775 } 2776 2777 ppgtt->base.clear_range(&ppgtt->base, 2778 ppgtt->base.start, 2779 ppgtt->base.total, 2780 true); 2781 2782 dev_priv->mm.aliasing_ppgtt = ppgtt; 2783 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); 2784 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; 2785 } 2786 2787 return 0; 2788 } 2789 2790 void i915_gem_init_global_gtt(struct drm_device *dev) 2791 { 2792 struct drm_i915_private *dev_priv = dev->dev_private; 2793 u64 gtt_size, mappable_size; 2794 2795 gtt_size = dev_priv->gtt.base.total; 2796 mappable_size = dev_priv->gtt.mappable_end; 2797 2798 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 2799 } 2800 2801 void i915_global_gtt_cleanup(struct drm_device *dev) 2802 { 2803 struct drm_i915_private *dev_priv = dev->dev_private; 2804 struct i915_address_space *vm = &dev_priv->gtt.base; 2805 2806 if (dev_priv->mm.aliasing_ppgtt) { 2807 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2808 2809 ppgtt->base.cleanup(&ppgtt->base); 2810 } 2811 2812 if (drm_mm_initialized(&vm->mm)) { 2813 if (intel_vgpu_active(dev)) 2814 intel_vgt_deballoon(); 2815 2816 drm_mm_takedown(&vm->mm); 2817 list_del(&vm->global_link); 2818 } 2819 2820 vm->cleanup(vm); 2821 } 2822 2823 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 2824 { 2825 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 2826 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 2827 return snb_gmch_ctl << 20; 2828 } 2829 2830 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 2831 { 2832 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 2833 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 2834 if (bdw_gmch_ctl) 2835 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 2836 2837 #ifdef CONFIG_X86_32 2838 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 2839 if (bdw_gmch_ctl > 4) 2840 bdw_gmch_ctl = 4; 2841 #endif 2842 2843 return bdw_gmch_ctl << 20; 2844 } 2845 2846 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 2847 { 2848 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 2849 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 2850 2851 if (gmch_ctrl) 2852 return 1 << (20 + gmch_ctrl); 2853 2854 return 0; 2855 } 2856 2857 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 2858 { 2859 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 2860 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 2861 return snb_gmch_ctl << 25; /* 32 MB units */ 2862 } 2863 2864 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 2865 { 2866 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2867 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 2868 return bdw_gmch_ctl << 25; /* 32 MB units */ 2869 } 2870 2871 static size_t chv_get_stolen_size(u16 gmch_ctrl) 2872 { 2873 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 2874 gmch_ctrl &= SNB_GMCH_GMS_MASK; 2875 2876 /* 2877 * 0x0 to 0x10: 32MB increments starting at 0MB 2878 * 0x11 to 0x16: 4MB increments starting at 8MB 2879 * 0x17 to 0x1d: 4MB increments start at 36MB 2880 */ 2881 if (gmch_ctrl < 0x11) 2882 return gmch_ctrl << 25; 2883 else if (gmch_ctrl < 0x17) 2884 return (gmch_ctrl - 0x11 + 2) << 22; 2885 else 2886 return (gmch_ctrl - 0x17 + 9) << 22; 2887 } 2888 2889 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 2890 { 2891 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2892 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 2893 2894 if (gen9_gmch_ctl < 0xf0) 2895 return gen9_gmch_ctl << 25; /* 32 MB units */ 2896 else 2897 /* 4MB increments starting at 0xf0 for 4MB */ 2898 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 2899 } 2900 2901 static int ggtt_probe_common(struct drm_device *dev, 2902 size_t gtt_size) 2903 { 2904 struct drm_i915_private *dev_priv = dev->dev_private; 2905 struct i915_page_scratch *scratch_page; 2906 phys_addr_t gtt_phys_addr; 2907 2908 /* For Modern GENs the PTEs and register space are split in the BAR */ 2909 gtt_phys_addr = pci_resource_start(dev->pdev, 0) + 2910 (pci_resource_len(dev->pdev, 0) / 2); 2911 2912 /* 2913 * On BXT writes larger than 64 bit to the GTT pagetable range will be 2914 * dropped. For WC mappings in general we have 64 byte burst writes 2915 * when the WC buffer is flushed, so we can't use it, but have to 2916 * resort to an uncached mapping. The WC issue is easily caught by the 2917 * readback check when writing GTT PTE entries. 2918 */ 2919 if (IS_BROXTON(dev)) 2920 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); 2921 else 2922 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); 2923 if (!dev_priv->gtt.gsm) { 2924 DRM_ERROR("Failed to map the gtt page table\n"); 2925 return -ENOMEM; 2926 } 2927 2928 scratch_page = alloc_scratch_page(dev); 2929 if (IS_ERR(scratch_page)) { 2930 DRM_ERROR("Scratch setup failed\n"); 2931 /* iounmap will also get called at remove, but meh */ 2932 iounmap(dev_priv->gtt.gsm); 2933 return PTR_ERR(scratch_page); 2934 } 2935 2936 dev_priv->gtt.base.scratch_page = scratch_page; 2937 2938 return 0; 2939 } 2940 2941 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 2942 * bits. When using advanced contexts each context stores its own PAT, but 2943 * writing this data shouldn't be harmful even in those cases. */ 2944 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 2945 { 2946 uint64_t pat; 2947 2948 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 2949 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 2950 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 2951 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 2952 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 2953 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 2954 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 2955 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 2956 2957 if (!USES_PPGTT(dev_priv->dev)) 2958 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 2959 * so RTL will always use the value corresponding to 2960 * pat_sel = 000". 2961 * So let's disable cache for GGTT to avoid screen corruptions. 2962 * MOCS still can be used though. 2963 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 2964 * before this patch, i.e. the same uncached + snooping access 2965 * like on gen6/7 seems to be in effect. 2966 * - So this just fixes blitter/render access. Again it looks 2967 * like it's not just uncached access, but uncached + snooping. 2968 * So we can still hold onto all our assumptions wrt cpu 2969 * clflushing on LLC machines. 2970 */ 2971 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 2972 2973 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 2974 * write would work. */ 2975 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2976 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2977 } 2978 2979 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 2980 { 2981 uint64_t pat; 2982 2983 /* 2984 * Map WB on BDW to snooped on CHV. 2985 * 2986 * Only the snoop bit has meaning for CHV, the rest is 2987 * ignored. 2988 * 2989 * The hardware will never snoop for certain types of accesses: 2990 * - CPU GTT (GMADR->GGTT->no snoop->memory) 2991 * - PPGTT page tables 2992 * - some other special cycles 2993 * 2994 * As with BDW, we also need to consider the following for GT accesses: 2995 * "For GGTT, there is NO pat_sel[2:0] from the entry, 2996 * so RTL will always use the value corresponding to 2997 * pat_sel = 000". 2998 * Which means we must set the snoop bit in PAT entry 0 2999 * in order to keep the global status page working. 3000 */ 3001 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 3002 GEN8_PPAT(1, 0) | 3003 GEN8_PPAT(2, 0) | 3004 GEN8_PPAT(3, 0) | 3005 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 3006 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 3007 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 3008 GEN8_PPAT(7, CHV_PPAT_SNOOP); 3009 3010 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 3011 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 3012 } 3013 3014 static int gen8_gmch_probe(struct drm_device *dev, 3015 u64 *gtt_total, 3016 size_t *stolen, 3017 phys_addr_t *mappable_base, 3018 u64 *mappable_end) 3019 { 3020 struct drm_i915_private *dev_priv = dev->dev_private; 3021 u64 gtt_size; 3022 u16 snb_gmch_ctl; 3023 int ret; 3024 3025 /* TODO: We're not aware of mappable constraints on gen8 yet */ 3026 *mappable_base = pci_resource_start(dev->pdev, 2); 3027 *mappable_end = pci_resource_len(dev->pdev, 2); 3028 3029 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) 3030 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); 3031 3032 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3033 3034 if (INTEL_INFO(dev)->gen >= 9) { 3035 *stolen = gen9_get_stolen_size(snb_gmch_ctl); 3036 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 3037 } else if (IS_CHERRYVIEW(dev)) { 3038 *stolen = chv_get_stolen_size(snb_gmch_ctl); 3039 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); 3040 } else { 3041 *stolen = gen8_get_stolen_size(snb_gmch_ctl); 3042 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 3043 } 3044 3045 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 3046 3047 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3048 chv_setup_private_ppat(dev_priv); 3049 else 3050 bdw_setup_private_ppat(dev_priv); 3051 3052 ret = ggtt_probe_common(dev, gtt_size); 3053 3054 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; 3055 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; 3056 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3057 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3058 3059 if (IS_CHERRYVIEW(dev_priv)) 3060 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries__BKL; 3061 3062 return ret; 3063 } 3064 3065 static int gen6_gmch_probe(struct drm_device *dev, 3066 u64 *gtt_total, 3067 size_t *stolen, 3068 phys_addr_t *mappable_base, 3069 u64 *mappable_end) 3070 { 3071 struct drm_i915_private *dev_priv = dev->dev_private; 3072 unsigned int gtt_size; 3073 u16 snb_gmch_ctl; 3074 int ret; 3075 3076 *mappable_base = pci_resource_start(dev->pdev, 2); 3077 *mappable_end = pci_resource_len(dev->pdev, 2); 3078 3079 /* 64/512MB is the current min/max we actually know of, but this is just 3080 * a coarse sanity check. 3081 */ 3082 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 3083 DRM_ERROR("Unknown GMADR size (%llx)\n", 3084 dev_priv->gtt.mappable_end); 3085 return -ENXIO; 3086 } 3087 3088 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 3089 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 3090 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3091 3092 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 3093 3094 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 3095 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 3096 3097 ret = ggtt_probe_common(dev, gtt_size); 3098 3099 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; 3100 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; 3101 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3102 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3103 3104 return ret; 3105 } 3106 3107 static void gen6_gmch_remove(struct i915_address_space *vm) 3108 { 3109 3110 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); 3111 3112 iounmap(gtt->gsm); 3113 free_scratch_page(vm->dev, vm->scratch_page); 3114 } 3115 3116 static int i915_gmch_probe(struct drm_device *dev, 3117 u64 *gtt_total, 3118 size_t *stolen, 3119 phys_addr_t *mappable_base, 3120 u64 *mappable_end) 3121 { 3122 struct drm_i915_private *dev_priv = dev->dev_private; 3123 int ret; 3124 3125 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); 3126 if (!ret) { 3127 DRM_ERROR("failed to set up gmch\n"); 3128 return -EIO; 3129 } 3130 3131 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); 3132 3133 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); 3134 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; 3135 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; 3136 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3137 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3138 3139 if (unlikely(dev_priv->gtt.do_idle_maps)) 3140 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 3141 3142 return 0; 3143 } 3144 3145 static void i915_gmch_remove(struct i915_address_space *vm) 3146 { 3147 intel_gmch_remove(); 3148 } 3149 3150 int i915_gem_gtt_init(struct drm_device *dev) 3151 { 3152 struct drm_i915_private *dev_priv = dev->dev_private; 3153 struct i915_gtt *gtt = &dev_priv->gtt; 3154 int ret; 3155 3156 if (INTEL_INFO(dev)->gen <= 5) { 3157 gtt->gtt_probe = i915_gmch_probe; 3158 gtt->base.cleanup = i915_gmch_remove; 3159 } else if (INTEL_INFO(dev)->gen < 8) { 3160 gtt->gtt_probe = gen6_gmch_probe; 3161 gtt->base.cleanup = gen6_gmch_remove; 3162 if (IS_HASWELL(dev) && dev_priv->ellc_size) 3163 gtt->base.pte_encode = iris_pte_encode; 3164 else if (IS_HASWELL(dev)) 3165 gtt->base.pte_encode = hsw_pte_encode; 3166 else if (IS_VALLEYVIEW(dev)) 3167 gtt->base.pte_encode = byt_pte_encode; 3168 else if (INTEL_INFO(dev)->gen >= 7) 3169 gtt->base.pte_encode = ivb_pte_encode; 3170 else 3171 gtt->base.pte_encode = snb_pte_encode; 3172 } else { 3173 dev_priv->gtt.gtt_probe = gen8_gmch_probe; 3174 dev_priv->gtt.base.cleanup = gen6_gmch_remove; 3175 } 3176 3177 gtt->base.dev = dev; 3178 3179 ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, 3180 >t->mappable_base, >t->mappable_end); 3181 if (ret) 3182 return ret; 3183 3184 /* GMADR is the PCI mmio aperture into the global GTT. */ 3185 DRM_INFO("Memory usable by graphics device = %lluM\n", 3186 gtt->base.total >> 20); 3187 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); 3188 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); 3189 #ifdef CONFIG_INTEL_IOMMU 3190 if (intel_iommu_gfx_mapped) 3191 DRM_INFO("VT-d active for gfx access\n"); 3192 #endif 3193 /* 3194 * i915.enable_ppgtt is read-only, so do an early pass to validate the 3195 * user's requested state against the hardware/driver capabilities. We 3196 * do this now so that we can print out any log messages once rather 3197 * than every time we check intel_enable_ppgtt(). 3198 */ 3199 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); 3200 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 3201 3202 return 0; 3203 } 3204 3205 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 3206 { 3207 struct drm_i915_private *dev_priv = dev->dev_private; 3208 struct drm_i915_gem_object *obj; 3209 struct i915_address_space *vm; 3210 struct i915_vma *vma; 3211 bool flush; 3212 3213 i915_check_and_clear_faults(dev); 3214 3215 /* First fill our portion of the GTT with scratch pages */ 3216 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 3217 dev_priv->gtt.base.start, 3218 dev_priv->gtt.base.total, 3219 true); 3220 3221 /* Cache flush objects bound into GGTT and rebind them. */ 3222 vm = &dev_priv->gtt.base; 3223 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 3224 flush = false; 3225 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3226 if (vma->vm != vm) 3227 continue; 3228 3229 WARN_ON(i915_vma_bind(vma, obj->cache_level, 3230 PIN_UPDATE)); 3231 3232 flush = true; 3233 } 3234 3235 if (flush) 3236 i915_gem_clflush_object(obj, obj->pin_display); 3237 } 3238 3239 if (INTEL_INFO(dev)->gen >= 8) { 3240 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3241 chv_setup_private_ppat(dev_priv); 3242 else 3243 bdw_setup_private_ppat(dev_priv); 3244 3245 return; 3246 } 3247 3248 if (USES_PPGTT(dev)) { 3249 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3250 /* TODO: Perhaps it shouldn't be gen6 specific */ 3251 3252 struct i915_hw_ppgtt *ppgtt = 3253 container_of(vm, struct i915_hw_ppgtt, 3254 base); 3255 3256 if (i915_is_ggtt(vm)) 3257 ppgtt = dev_priv->mm.aliasing_ppgtt; 3258 3259 gen6_write_page_range(dev_priv, &ppgtt->pd, 3260 0, ppgtt->base.total); 3261 } 3262 } 3263 3264 i915_ggtt_flush(dev_priv); 3265 } 3266 3267 static struct i915_vma * 3268 __i915_gem_vma_create(struct drm_i915_gem_object *obj, 3269 struct i915_address_space *vm, 3270 const struct i915_ggtt_view *ggtt_view) 3271 { 3272 struct i915_vma *vma; 3273 3274 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 3275 return ERR_PTR(-EINVAL); 3276 3277 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); 3278 if (vma == NULL) 3279 return ERR_PTR(-ENOMEM); 3280 3281 INIT_LIST_HEAD(&vma->vma_link); 3282 INIT_LIST_HEAD(&vma->mm_list); 3283 INIT_LIST_HEAD(&vma->exec_list); 3284 vma->vm = vm; 3285 vma->obj = obj; 3286 3287 if (i915_is_ggtt(vm)) 3288 vma->ggtt_view = *ggtt_view; 3289 3290 list_add_tail(&vma->vma_link, &obj->vma_list); 3291 if (!i915_is_ggtt(vm)) 3292 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 3293 3294 return vma; 3295 } 3296 3297 struct i915_vma * 3298 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 3299 struct i915_address_space *vm) 3300 { 3301 struct i915_vma *vma; 3302 3303 vma = i915_gem_obj_to_vma(obj, vm); 3304 if (!vma) 3305 vma = __i915_gem_vma_create(obj, vm, 3306 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); 3307 3308 return vma; 3309 } 3310 3311 struct i915_vma * 3312 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, 3313 const struct i915_ggtt_view *view) 3314 { 3315 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 3316 struct i915_vma *vma; 3317 3318 if (WARN_ON(!view)) 3319 return ERR_PTR(-EINVAL); 3320 3321 vma = i915_gem_obj_to_ggtt_view(obj, view); 3322 3323 if (IS_ERR(vma)) 3324 return vma; 3325 3326 if (!vma) 3327 vma = __i915_gem_vma_create(obj, ggtt, view); 3328 3329 return vma; 3330 3331 } 3332 3333 static struct scatterlist * 3334 rotate_pages(const dma_addr_t *in, unsigned int offset, 3335 unsigned int width, unsigned int height, 3336 struct sg_table *st, struct scatterlist *sg) 3337 { 3338 unsigned int column, row; 3339 unsigned int src_idx; 3340 3341 if (!sg) { 3342 st->nents = 0; 3343 sg = st->sgl; 3344 } 3345 3346 for (column = 0; column < width; column++) { 3347 src_idx = width * (height - 1) + column; 3348 for (row = 0; row < height; row++) { 3349 st->nents++; 3350 /* We don't need the pages, but need to initialize 3351 * the entries so the sg list can be happily traversed. 3352 * The only thing we need are DMA addresses. 3353 */ 3354 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3355 sg_dma_address(sg) = in[offset + src_idx]; 3356 sg_dma_len(sg) = PAGE_SIZE; 3357 sg = sg_next(sg); 3358 src_idx -= width; 3359 } 3360 } 3361 3362 return sg; 3363 } 3364 3365 static struct sg_table * 3366 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, 3367 struct drm_i915_gem_object *obj) 3368 { 3369 struct intel_rotation_info *rot_info = &ggtt_view->params.rotation_info; 3370 unsigned int size_pages = rot_info->size >> PAGE_SHIFT; 3371 unsigned int size_pages_uv; 3372 struct sg_page_iter sg_iter; 3373 unsigned long i; 3374 dma_addr_t *page_addr_list; 3375 struct sg_table *st; 3376 unsigned int uv_start_page; 3377 struct scatterlist *sg; 3378 int ret = -ENOMEM; 3379 3380 /* Allocate a temporary list of source pages for random access. */ 3381 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, 3382 sizeof(dma_addr_t)); 3383 if (!page_addr_list) 3384 return ERR_PTR(ret); 3385 3386 /* Account for UV plane with NV12. */ 3387 if (rot_info->pixel_format == DRM_FORMAT_NV12) 3388 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; 3389 else 3390 size_pages_uv = 0; 3391 3392 /* Allocate target SG list. */ 3393 st = kmalloc(sizeof(*st), GFP_KERNEL); 3394 if (!st) 3395 goto err_st_alloc; 3396 3397 ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); 3398 if (ret) 3399 goto err_sg_alloc; 3400 3401 /* Populate source page list from the object. */ 3402 i = 0; 3403 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 3404 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); 3405 i++; 3406 } 3407 3408 /* Rotate the pages. */ 3409 sg = rotate_pages(page_addr_list, 0, 3410 rot_info->width_pages, rot_info->height_pages, 3411 st, NULL); 3412 3413 /* Append the UV plane if NV12. */ 3414 if (rot_info->pixel_format == DRM_FORMAT_NV12) { 3415 uv_start_page = size_pages; 3416 3417 /* Check for tile-row un-alignment. */ 3418 if (offset_in_page(rot_info->uv_offset)) 3419 uv_start_page--; 3420 3421 rot_info->uv_start_page = uv_start_page; 3422 3423 rotate_pages(page_addr_list, uv_start_page, 3424 rot_info->width_pages_uv, 3425 rot_info->height_pages_uv, 3426 st, sg); 3427 } 3428 3429 DRM_DEBUG_KMS( 3430 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", 3431 obj->base.size, rot_info->pitch, rot_info->height, 3432 rot_info->pixel_format, rot_info->width_pages, 3433 rot_info->height_pages, size_pages + size_pages_uv, 3434 size_pages); 3435 3436 drm_free_large(page_addr_list); 3437 3438 return st; 3439 3440 err_sg_alloc: 3441 kfree(st); 3442 err_st_alloc: 3443 drm_free_large(page_addr_list); 3444 3445 DRM_DEBUG_KMS( 3446 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", 3447 obj->base.size, ret, rot_info->pitch, rot_info->height, 3448 rot_info->pixel_format, rot_info->width_pages, 3449 rot_info->height_pages, size_pages + size_pages_uv, 3450 size_pages); 3451 return ERR_PTR(ret); 3452 } 3453 3454 static struct sg_table * 3455 intel_partial_pages(const struct i915_ggtt_view *view, 3456 struct drm_i915_gem_object *obj) 3457 { 3458 struct sg_table *st; 3459 struct scatterlist *sg; 3460 struct sg_page_iter obj_sg_iter; 3461 int ret = -ENOMEM; 3462 3463 st = kmalloc(sizeof(*st), GFP_KERNEL); 3464 if (!st) 3465 goto err_st_alloc; 3466 3467 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); 3468 if (ret) 3469 goto err_sg_alloc; 3470 3471 sg = st->sgl; 3472 st->nents = 0; 3473 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, 3474 view->params.partial.offset) 3475 { 3476 if (st->nents >= view->params.partial.size) 3477 break; 3478 3479 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3480 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); 3481 sg_dma_len(sg) = PAGE_SIZE; 3482 3483 sg = sg_next(sg); 3484 st->nents++; 3485 } 3486 3487 return st; 3488 3489 err_sg_alloc: 3490 kfree(st); 3491 err_st_alloc: 3492 return ERR_PTR(ret); 3493 } 3494 3495 static int 3496 i915_get_ggtt_vma_pages(struct i915_vma *vma) 3497 { 3498 int ret = 0; 3499 3500 if (vma->ggtt_view.pages) 3501 return 0; 3502 3503 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 3504 vma->ggtt_view.pages = vma->obj->pages; 3505 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 3506 vma->ggtt_view.pages = 3507 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); 3508 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) 3509 vma->ggtt_view.pages = 3510 intel_partial_pages(&vma->ggtt_view, vma->obj); 3511 else 3512 WARN_ONCE(1, "GGTT view %u not implemented!\n", 3513 vma->ggtt_view.type); 3514 3515 if (!vma->ggtt_view.pages) { 3516 DRM_ERROR("Failed to get pages for GGTT view type %u!\n", 3517 vma->ggtt_view.type); 3518 ret = -EINVAL; 3519 } else if (IS_ERR(vma->ggtt_view.pages)) { 3520 ret = PTR_ERR(vma->ggtt_view.pages); 3521 vma->ggtt_view.pages = NULL; 3522 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 3523 vma->ggtt_view.type, ret); 3524 } 3525 3526 return ret; 3527 } 3528 3529 /** 3530 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 3531 * @vma: VMA to map 3532 * @cache_level: mapping cache level 3533 * @flags: flags like global or local mapping 3534 * 3535 * DMA addresses are taken from the scatter-gather table of this object (or of 3536 * this VMA in case of non-default GGTT views) and PTE entries set up. 3537 * Note that DMA addresses are also the only part of the SG table we care about. 3538 */ 3539 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 3540 u32 flags) 3541 { 3542 int ret; 3543 u32 bind_flags; 3544 3545 if (WARN_ON(flags == 0)) 3546 return -EINVAL; 3547 3548 bind_flags = 0; 3549 if (flags & PIN_GLOBAL) 3550 bind_flags |= GLOBAL_BIND; 3551 if (flags & PIN_USER) 3552 bind_flags |= LOCAL_BIND; 3553 3554 if (flags & PIN_UPDATE) 3555 bind_flags |= vma->bound; 3556 else 3557 bind_flags &= ~vma->bound; 3558 3559 if (bind_flags == 0) 3560 return 0; 3561 3562 if (vma->bound == 0 && vma->vm->allocate_va_range) { 3563 trace_i915_va_alloc(vma->vm, 3564 vma->node.start, 3565 vma->node.size, 3566 VM_TO_TRACE_NAME(vma->vm)); 3567 3568 /* XXX: i915_vma_pin() will fix this +- hack */ 3569 vma->pin_count++; 3570 ret = vma->vm->allocate_va_range(vma->vm, 3571 vma->node.start, 3572 vma->node.size); 3573 vma->pin_count--; 3574 if (ret) 3575 return ret; 3576 } 3577 3578 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 3579 if (ret) 3580 return ret; 3581 3582 vma->bound |= bind_flags; 3583 3584 return 0; 3585 } 3586 3587 /** 3588 * i915_ggtt_view_size - Get the size of a GGTT view. 3589 * @obj: Object the view is of. 3590 * @view: The view in question. 3591 * 3592 * @return The size of the GGTT view in bytes. 3593 */ 3594 size_t 3595 i915_ggtt_view_size(struct drm_i915_gem_object *obj, 3596 const struct i915_ggtt_view *view) 3597 { 3598 if (view->type == I915_GGTT_VIEW_NORMAL) { 3599 return obj->base.size; 3600 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 3601 return view->params.rotation_info.size; 3602 } else if (view->type == I915_GGTT_VIEW_PARTIAL) { 3603 return view->params.partial.size << PAGE_SHIFT; 3604 } else { 3605 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type); 3606 return obj->base.size; 3607 } 3608 } 3609