1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/stop_machine.h> 7 8 #include <asm/set_memory.h> 9 #include <asm/smp.h> 10 11 #include <drm/i915_drm.h> 12 13 #include "intel_gt.h" 14 #include "i915_drv.h" 15 #include "i915_scatterlist.h" 16 #include "i915_vgpu.h" 17 18 #include "intel_gtt.h" 19 20 static int 21 i915_get_ggtt_vma_pages(struct i915_vma *vma); 22 23 static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 24 unsigned long color, 25 u64 *start, 26 u64 *end) 27 { 28 if (i915_node_color_differs(node, color)) 29 *start += I915_GTT_PAGE_SIZE; 30 31 /* 32 * Also leave a space between the unallocated reserved node after the 33 * GTT and any objects within the GTT, i.e. we use the color adjustment 34 * to insert a guard page to prevent prefetches crossing over the 35 * GTT boundary. 36 */ 37 node = list_next_entry(node, node_list); 38 if (node->color != color) 39 *end -= I915_GTT_PAGE_SIZE; 40 } 41 42 static int ggtt_init_hw(struct i915_ggtt *ggtt) 43 { 44 struct drm_i915_private *i915 = ggtt->vm.i915; 45 46 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 47 48 ggtt->vm.is_ggtt = true; 49 50 /* Only VLV supports read-only GGTT mappings */ 51 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 52 53 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 54 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 55 56 if (ggtt->mappable_end) { 57 if (!io_mapping_init_wc(&ggtt->iomap, 58 ggtt->gmadr.start, 59 ggtt->mappable_end)) { 60 ggtt->vm.cleanup(&ggtt->vm); 61 return -EIO; 62 } 63 64 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 65 ggtt->mappable_end); 66 } 67 68 intel_ggtt_init_fences(ggtt); 69 70 return 0; 71 } 72 73 /** 74 * i915_ggtt_init_hw - Initialize GGTT hardware 75 * @i915: i915 device 76 */ 77 int i915_ggtt_init_hw(struct drm_i915_private *i915) 78 { 79 int ret; 80 81 /* 82 * Note that we use page colouring to enforce a guard page at the 83 * end of the address space. This is required as the CS may prefetch 84 * beyond the end of the batch buffer, across the page boundary, 85 * and beyond the end of the GTT if we do not provide a guard. 86 */ 87 ret = ggtt_init_hw(&i915->ggtt); 88 if (ret) 89 return ret; 90 91 return 0; 92 } 93 94 /* 95 * Certain Gen5 chipsets require require idling the GPU before 96 * unmapping anything from the GTT when VT-d is enabled. 97 */ 98 static bool needs_idle_maps(struct drm_i915_private *i915) 99 { 100 /* 101 * Query intel_iommu to see if we need the workaround. Presumably that 102 * was loaded first. 103 */ 104 return IS_GEN(i915, 5) && IS_MOBILE(i915) && intel_vtd_active(); 105 } 106 107 void i915_ggtt_suspend(struct i915_ggtt *ggtt) 108 { 109 struct i915_vma *vma, *vn; 110 int open; 111 112 mutex_lock(&ggtt->vm.mutex); 113 114 /* Skip rewriting PTE on VMA unbind. */ 115 open = atomic_xchg(&ggtt->vm.open, 0); 116 117 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 118 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 119 i915_vma_wait_for_bind(vma); 120 121 if (i915_vma_is_pinned(vma)) 122 continue; 123 124 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 125 __i915_vma_evict(vma); 126 drm_mm_remove_node(&vma->node); 127 } 128 } 129 130 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 131 ggtt->invalidate(ggtt); 132 atomic_set(&ggtt->vm.open, open); 133 134 mutex_unlock(&ggtt->vm.mutex); 135 136 intel_gt_check_and_clear_faults(ggtt->vm.gt); 137 } 138 139 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 140 { 141 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 142 143 spin_lock_irq(&uncore->lock); 144 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 145 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 146 spin_unlock_irq(&uncore->lock); 147 } 148 149 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 150 { 151 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 152 153 /* 154 * Note that as an uncached mmio write, this will flush the 155 * WCB of the writes into the GGTT before it triggers the invalidate. 156 */ 157 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 158 } 159 160 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 161 { 162 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 163 struct drm_i915_private *i915 = ggtt->vm.i915; 164 165 gen8_ggtt_invalidate(ggtt); 166 167 if (INTEL_GEN(i915) >= 12) 168 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR, 169 GEN12_GUC_TLB_INV_CR_INVALIDATE); 170 else 171 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE); 172 } 173 174 static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt) 175 { 176 intel_gtt_chipset_flush(); 177 } 178 179 static u64 gen8_ggtt_pte_encode(dma_addr_t addr, 180 enum i915_cache_level level, 181 u32 flags) 182 { 183 return addr | _PAGE_PRESENT; 184 } 185 186 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 187 { 188 writeq(pte, addr); 189 } 190 191 static void gen8_ggtt_insert_page(struct i915_address_space *vm, 192 dma_addr_t addr, 193 u64 offset, 194 enum i915_cache_level level, 195 u32 unused) 196 { 197 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 198 gen8_pte_t __iomem *pte = 199 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 200 201 gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, 0)); 202 203 ggtt->invalidate(ggtt); 204 } 205 206 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 207 struct i915_vma *vma, 208 enum i915_cache_level level, 209 u32 flags) 210 { 211 const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, 0); 212 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 213 gen8_pte_t __iomem *gte; 214 gen8_pte_t __iomem *end; 215 struct sgt_iter iter; 216 dma_addr_t addr; 217 218 /* 219 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 220 * not to allow the user to override access to a read only page. 221 */ 222 223 gte = (gen8_pte_t __iomem *)ggtt->gsm; 224 gte += vma->node.start / I915_GTT_PAGE_SIZE; 225 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 226 227 for_each_sgt_daddr(addr, iter, vma->pages) 228 gen8_set_pte(gte++, pte_encode | addr); 229 GEM_BUG_ON(gte > end); 230 231 /* Fill the allocated but "unused" space beyond the end of the buffer */ 232 while (gte < end) 233 gen8_set_pte(gte++, vm->scratch[0]->encode); 234 235 /* 236 * We want to flush the TLBs only after we're certain all the PTE 237 * updates have finished. 238 */ 239 ggtt->invalidate(ggtt); 240 } 241 242 static void gen6_ggtt_insert_page(struct i915_address_space *vm, 243 dma_addr_t addr, 244 u64 offset, 245 enum i915_cache_level level, 246 u32 flags) 247 { 248 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 249 gen6_pte_t __iomem *pte = 250 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 251 252 iowrite32(vm->pte_encode(addr, level, flags), pte); 253 254 ggtt->invalidate(ggtt); 255 } 256 257 /* 258 * Binds an object into the global gtt with the specified cache level. 259 * The object will be accessible to the GPU via commands whose operands 260 * reference offsets within the global GTT as well as accessible by the GPU 261 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 262 */ 263 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 264 struct i915_vma *vma, 265 enum i915_cache_level level, 266 u32 flags) 267 { 268 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 269 gen6_pte_t __iomem *gte; 270 gen6_pte_t __iomem *end; 271 struct sgt_iter iter; 272 dma_addr_t addr; 273 274 gte = (gen6_pte_t __iomem *)ggtt->gsm; 275 gte += vma->node.start / I915_GTT_PAGE_SIZE; 276 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 277 278 for_each_sgt_daddr(addr, iter, vma->pages) 279 iowrite32(vm->pte_encode(addr, level, flags), gte++); 280 GEM_BUG_ON(gte > end); 281 282 /* Fill the allocated but "unused" space beyond the end of the buffer */ 283 while (gte < end) 284 iowrite32(vm->scratch[0]->encode, gte++); 285 286 /* 287 * We want to flush the TLBs only after we're certain all the PTE 288 * updates have finished. 289 */ 290 ggtt->invalidate(ggtt); 291 } 292 293 static void nop_clear_range(struct i915_address_space *vm, 294 u64 start, u64 length) 295 { 296 } 297 298 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 299 u64 start, u64 length) 300 { 301 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 302 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 303 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 304 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 305 gen8_pte_t __iomem *gtt_base = 306 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 307 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 308 int i; 309 310 if (WARN(num_entries > max_entries, 311 "First entry = %d; Num entries = %d (max=%d)\n", 312 first_entry, num_entries, max_entries)) 313 num_entries = max_entries; 314 315 for (i = 0; i < num_entries; i++) 316 gen8_set_pte(>t_base[i], scratch_pte); 317 } 318 319 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 320 { 321 /* 322 * Make sure the internal GAM fifo has been cleared of all GTT 323 * writes before exiting stop_machine(). This guarantees that 324 * any aperture accesses waiting to start in another process 325 * cannot back up behind the GTT writes causing a hang. 326 * The register can be any arbitrary GAM register. 327 */ 328 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 329 } 330 331 struct insert_page { 332 struct i915_address_space *vm; 333 dma_addr_t addr; 334 u64 offset; 335 enum i915_cache_level level; 336 }; 337 338 static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 339 { 340 struct insert_page *arg = _arg; 341 342 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 343 bxt_vtd_ggtt_wa(arg->vm); 344 345 return 0; 346 } 347 348 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 349 dma_addr_t addr, 350 u64 offset, 351 enum i915_cache_level level, 352 u32 unused) 353 { 354 struct insert_page arg = { vm, addr, offset, level }; 355 356 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 357 } 358 359 struct insert_entries { 360 struct i915_address_space *vm; 361 struct i915_vma *vma; 362 enum i915_cache_level level; 363 u32 flags; 364 }; 365 366 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 367 { 368 struct insert_entries *arg = _arg; 369 370 gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags); 371 bxt_vtd_ggtt_wa(arg->vm); 372 373 return 0; 374 } 375 376 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 377 struct i915_vma *vma, 378 enum i915_cache_level level, 379 u32 flags) 380 { 381 struct insert_entries arg = { vm, vma, level, flags }; 382 383 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 384 } 385 386 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 387 u64 start, u64 length) 388 { 389 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 390 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 391 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 392 gen6_pte_t scratch_pte, __iomem *gtt_base = 393 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 394 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 395 int i; 396 397 if (WARN(num_entries > max_entries, 398 "First entry = %d; Num entries = %d (max=%d)\n", 399 first_entry, num_entries, max_entries)) 400 num_entries = max_entries; 401 402 scratch_pte = vm->scratch[0]->encode; 403 for (i = 0; i < num_entries; i++) 404 iowrite32(scratch_pte, >t_base[i]); 405 } 406 407 static void i915_ggtt_insert_page(struct i915_address_space *vm, 408 dma_addr_t addr, 409 u64 offset, 410 enum i915_cache_level cache_level, 411 u32 unused) 412 { 413 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 414 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 415 416 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); 417 } 418 419 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 420 struct i915_vma *vma, 421 enum i915_cache_level cache_level, 422 u32 unused) 423 { 424 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 425 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 426 427 intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT, 428 flags); 429 } 430 431 static void i915_ggtt_clear_range(struct i915_address_space *vm, 432 u64 start, u64 length) 433 { 434 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); 435 } 436 437 static void ggtt_bind_vma(struct i915_address_space *vm, 438 struct i915_vm_pt_stash *stash, 439 struct i915_vma *vma, 440 enum i915_cache_level cache_level, 441 u32 flags) 442 { 443 struct drm_i915_gem_object *obj = vma->obj; 444 u32 pte_flags; 445 446 if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK)) 447 return; 448 449 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 450 pte_flags = 0; 451 if (i915_gem_object_is_readonly(obj)) 452 pte_flags |= PTE_READ_ONLY; 453 454 vm->insert_entries(vm, vma, cache_level, pte_flags); 455 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; 456 } 457 458 static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) 459 { 460 vm->clear_range(vm, vma->node.start, vma->size); 461 } 462 463 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 464 { 465 u64 size; 466 int ret; 467 468 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 469 return 0; 470 471 GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); 472 size = ggtt->vm.total - GUC_GGTT_TOP; 473 474 ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size, 475 GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, 476 PIN_NOEVICT); 477 if (ret) 478 drm_dbg(&ggtt->vm.i915->drm, 479 "Failed to reserve top of GGTT for GuC\n"); 480 481 return ret; 482 } 483 484 static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 485 { 486 if (drm_mm_node_allocated(&ggtt->uc_fw)) 487 drm_mm_remove_node(&ggtt->uc_fw); 488 } 489 490 static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 491 { 492 ggtt_release_guc_top(ggtt); 493 if (drm_mm_node_allocated(&ggtt->error_capture)) 494 drm_mm_remove_node(&ggtt->error_capture); 495 mutex_destroy(&ggtt->error_mutex); 496 } 497 498 static int init_ggtt(struct i915_ggtt *ggtt) 499 { 500 /* 501 * Let GEM Manage all of the aperture. 502 * 503 * However, leave one page at the end still bound to the scratch page. 504 * There are a number of places where the hardware apparently prefetches 505 * past the end of the object, and we've seen multiple hangs with the 506 * GPU head pointer stuck in a batchbuffer bound at the last page of the 507 * aperture. One page should be enough to keep any prefetching inside 508 * of the aperture. 509 */ 510 unsigned long hole_start, hole_end; 511 struct drm_mm_node *entry; 512 int ret; 513 514 /* 515 * GuC requires all resources that we're sharing with it to be placed in 516 * non-WOPCM memory. If GuC is not present or not in use we still need a 517 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 518 * why. 519 */ 520 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 521 intel_wopcm_guc_size(&ggtt->vm.i915->wopcm)); 522 523 ret = intel_vgt_balloon(ggtt); 524 if (ret) 525 return ret; 526 527 mutex_init(&ggtt->error_mutex); 528 if (ggtt->mappable_end) { 529 /* Reserve a mappable slot for our lockless error capture */ 530 ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, 531 &ggtt->error_capture, 532 PAGE_SIZE, 0, 533 I915_COLOR_UNEVICTABLE, 534 0, ggtt->mappable_end, 535 DRM_MM_INSERT_LOW); 536 if (ret) 537 return ret; 538 } 539 540 /* 541 * The upper portion of the GuC address space has a sizeable hole 542 * (several MB) that is inaccessible by GuC. Reserve this range within 543 * GGTT as it can comfortably hold GuC/HuC firmware images. 544 */ 545 ret = ggtt_reserve_guc_top(ggtt); 546 if (ret) 547 goto err; 548 549 /* Clear any non-preallocated blocks */ 550 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 551 drm_dbg_kms(&ggtt->vm.i915->drm, 552 "clearing unused GTT space: [%lx, %lx]\n", 553 hole_start, hole_end); 554 ggtt->vm.clear_range(&ggtt->vm, hole_start, 555 hole_end - hole_start); 556 } 557 558 /* And finally clear the reserved guard page */ 559 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 560 561 return 0; 562 563 err: 564 cleanup_init_ggtt(ggtt); 565 return ret; 566 } 567 568 static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 569 struct i915_vm_pt_stash *stash, 570 struct i915_vma *vma, 571 enum i915_cache_level cache_level, 572 u32 flags) 573 { 574 u32 pte_flags; 575 576 /* Currently applicable only to VLV */ 577 pte_flags = 0; 578 if (i915_gem_object_is_readonly(vma->obj)) 579 pte_flags |= PTE_READ_ONLY; 580 581 if (flags & I915_VMA_LOCAL_BIND) 582 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 583 stash, vma, cache_level, flags); 584 585 if (flags & I915_VMA_GLOBAL_BIND) 586 vm->insert_entries(vm, vma, cache_level, pte_flags); 587 } 588 589 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 590 struct i915_vma *vma) 591 { 592 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 593 vm->clear_range(vm, vma->node.start, vma->size); 594 595 if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) 596 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma); 597 } 598 599 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 600 { 601 struct i915_vm_pt_stash stash = {}; 602 struct i915_ppgtt *ppgtt; 603 int err; 604 605 ppgtt = i915_ppgtt_create(ggtt->vm.gt); 606 if (IS_ERR(ppgtt)) 607 return PTR_ERR(ppgtt); 608 609 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 610 err = -ENODEV; 611 goto err_ppgtt; 612 } 613 614 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 615 if (err) 616 goto err_ppgtt; 617 618 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); 619 if (err) 620 goto err_stash; 621 622 /* 623 * Note we only pre-allocate as far as the end of the global 624 * GTT. On 48b / 4-level page-tables, the difference is very, 625 * very significant! We have to preallocate as GVT/vgpu does 626 * not like the page directory disappearing. 627 */ 628 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 629 630 ggtt->alias = ppgtt; 631 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 632 633 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma); 634 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 635 636 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma); 637 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 638 639 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 640 return 0; 641 642 err_stash: 643 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 644 err_ppgtt: 645 i915_vm_put(&ppgtt->vm); 646 return err; 647 } 648 649 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 650 { 651 struct i915_ppgtt *ppgtt; 652 653 ppgtt = fetch_and_zero(&ggtt->alias); 654 if (!ppgtt) 655 return; 656 657 i915_vm_put(&ppgtt->vm); 658 659 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 660 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 661 } 662 663 int i915_init_ggtt(struct drm_i915_private *i915) 664 { 665 int ret; 666 667 ret = init_ggtt(&i915->ggtt); 668 if (ret) 669 return ret; 670 671 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 672 ret = init_aliasing_ppgtt(&i915->ggtt); 673 if (ret) 674 cleanup_init_ggtt(&i915->ggtt); 675 } 676 677 return 0; 678 } 679 680 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 681 { 682 struct i915_vma *vma, *vn; 683 684 atomic_set(&ggtt->vm.open, 0); 685 686 rcu_barrier(); /* flush the RCU'ed__i915_vm_release */ 687 flush_workqueue(ggtt->vm.i915->wq); 688 689 mutex_lock(&ggtt->vm.mutex); 690 691 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) 692 WARN_ON(__i915_vma_unbind(vma)); 693 694 if (drm_mm_node_allocated(&ggtt->error_capture)) 695 drm_mm_remove_node(&ggtt->error_capture); 696 mutex_destroy(&ggtt->error_mutex); 697 698 ggtt_release_guc_top(ggtt); 699 intel_vgt_deballoon(ggtt); 700 701 ggtt->vm.cleanup(&ggtt->vm); 702 703 mutex_unlock(&ggtt->vm.mutex); 704 i915_address_space_fini(&ggtt->vm); 705 706 arch_phys_wc_del(ggtt->mtrr); 707 708 if (ggtt->iomap.size) 709 io_mapping_fini(&ggtt->iomap); 710 } 711 712 /** 713 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 714 * @i915: i915 device 715 */ 716 void i915_ggtt_driver_release(struct drm_i915_private *i915) 717 { 718 struct i915_ggtt *ggtt = &i915->ggtt; 719 720 fini_aliasing_ppgtt(ggtt); 721 722 intel_ggtt_fini_fences(ggtt); 723 ggtt_cleanup_hw(ggtt); 724 } 725 726 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 727 { 728 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 729 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 730 return snb_gmch_ctl << 20; 731 } 732 733 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 734 { 735 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 736 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 737 if (bdw_gmch_ctl) 738 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 739 740 #ifdef CONFIG_X86_32 741 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 742 if (bdw_gmch_ctl > 4) 743 bdw_gmch_ctl = 4; 744 #endif 745 746 return bdw_gmch_ctl << 20; 747 } 748 749 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 750 { 751 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 752 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 753 754 if (gmch_ctrl) 755 return 1 << (20 + gmch_ctrl); 756 757 return 0; 758 } 759 760 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 761 { 762 struct drm_i915_private *i915 = ggtt->vm.i915; 763 struct pci_dev *pdev = i915->drm.pdev; 764 phys_addr_t phys_addr; 765 int ret; 766 767 /* For Modern GENs the PTEs and register space are split in the BAR */ 768 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; 769 770 /* 771 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range 772 * will be dropped. For WC mappings in general we have 64 byte burst 773 * writes when the WC buffer is flushed, so we can't use it, but have to 774 * resort to an uncached mapping. The WC issue is easily caught by the 775 * readback check when writing GTT PTE entries. 776 */ 777 if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10) 778 ggtt->gsm = ioremap(phys_addr, size); 779 else 780 ggtt->gsm = ioremap_wc(phys_addr, size); 781 if (!ggtt->gsm) { 782 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 783 return -ENOMEM; 784 } 785 786 ret = setup_scratch_page(&ggtt->vm); 787 if (ret) { 788 drm_err(&i915->drm, "Scratch setup failed\n"); 789 /* iounmap will also get called at remove, but meh */ 790 iounmap(ggtt->gsm); 791 return ret; 792 } 793 794 ggtt->vm.scratch[0]->encode = 795 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 796 I915_CACHE_NONE, 0); 797 798 return 0; 799 } 800 801 int ggtt_set_pages(struct i915_vma *vma) 802 { 803 int ret; 804 805 GEM_BUG_ON(vma->pages); 806 807 ret = i915_get_ggtt_vma_pages(vma); 808 if (ret) 809 return ret; 810 811 vma->page_sizes = vma->obj->mm.page_sizes; 812 813 return 0; 814 } 815 816 static void gen6_gmch_remove(struct i915_address_space *vm) 817 { 818 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 819 820 iounmap(ggtt->gsm); 821 free_scratch(vm); 822 } 823 824 static struct resource pci_resource(struct pci_dev *pdev, int bar) 825 { 826 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar), 827 pci_resource_len(pdev, bar)); 828 } 829 830 static int gen8_gmch_probe(struct i915_ggtt *ggtt) 831 { 832 struct drm_i915_private *i915 = ggtt->vm.i915; 833 struct pci_dev *pdev = i915->drm.pdev; 834 unsigned int size; 835 u16 snb_gmch_ctl; 836 837 /* TODO: We're not aware of mappable constraints on gen8 yet */ 838 if (!IS_DGFX(i915)) { 839 ggtt->gmadr = pci_resource(pdev, 2); 840 ggtt->mappable_end = resource_size(&ggtt->gmadr); 841 } 842 843 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 844 if (IS_CHERRYVIEW(i915)) 845 size = chv_get_total_gtt_size(snb_gmch_ctl); 846 else 847 size = gen8_get_total_gtt_size(snb_gmch_ctl); 848 849 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 850 851 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 852 ggtt->vm.cleanup = gen6_gmch_remove; 853 ggtt->vm.insert_page = gen8_ggtt_insert_page; 854 ggtt->vm.clear_range = nop_clear_range; 855 if (intel_scanout_needs_vtd_wa(i915)) 856 ggtt->vm.clear_range = gen8_ggtt_clear_range; 857 858 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 859 860 /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ 861 if (intel_ggtt_update_needs_vtd_wa(i915) || 862 IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) { 863 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 864 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 865 ggtt->vm.bind_async_flags = 866 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 867 } 868 869 ggtt->invalidate = gen8_ggtt_invalidate; 870 871 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 872 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 873 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 874 ggtt->vm.vma_ops.clear_pages = clear_pages; 875 876 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 877 878 setup_private_pat(ggtt->vm.gt->uncore); 879 880 return ggtt_probe_common(ggtt, size); 881 } 882 883 static u64 snb_pte_encode(dma_addr_t addr, 884 enum i915_cache_level level, 885 u32 flags) 886 { 887 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 888 889 switch (level) { 890 case I915_CACHE_L3_LLC: 891 case I915_CACHE_LLC: 892 pte |= GEN6_PTE_CACHE_LLC; 893 break; 894 case I915_CACHE_NONE: 895 pte |= GEN6_PTE_UNCACHED; 896 break; 897 default: 898 MISSING_CASE(level); 899 } 900 901 return pte; 902 } 903 904 static u64 ivb_pte_encode(dma_addr_t addr, 905 enum i915_cache_level level, 906 u32 flags) 907 { 908 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 909 910 switch (level) { 911 case I915_CACHE_L3_LLC: 912 pte |= GEN7_PTE_CACHE_L3_LLC; 913 break; 914 case I915_CACHE_LLC: 915 pte |= GEN6_PTE_CACHE_LLC; 916 break; 917 case I915_CACHE_NONE: 918 pte |= GEN6_PTE_UNCACHED; 919 break; 920 default: 921 MISSING_CASE(level); 922 } 923 924 return pte; 925 } 926 927 static u64 byt_pte_encode(dma_addr_t addr, 928 enum i915_cache_level level, 929 u32 flags) 930 { 931 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 932 933 if (!(flags & PTE_READ_ONLY)) 934 pte |= BYT_PTE_WRITEABLE; 935 936 if (level != I915_CACHE_NONE) 937 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 938 939 return pte; 940 } 941 942 static u64 hsw_pte_encode(dma_addr_t addr, 943 enum i915_cache_level level, 944 u32 flags) 945 { 946 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 947 948 if (level != I915_CACHE_NONE) 949 pte |= HSW_WB_LLC_AGE3; 950 951 return pte; 952 } 953 954 static u64 iris_pte_encode(dma_addr_t addr, 955 enum i915_cache_level level, 956 u32 flags) 957 { 958 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 959 960 switch (level) { 961 case I915_CACHE_NONE: 962 break; 963 case I915_CACHE_WT: 964 pte |= HSW_WT_ELLC_LLC_AGE3; 965 break; 966 default: 967 pte |= HSW_WB_ELLC_LLC_AGE3; 968 break; 969 } 970 971 return pte; 972 } 973 974 static int gen6_gmch_probe(struct i915_ggtt *ggtt) 975 { 976 struct drm_i915_private *i915 = ggtt->vm.i915; 977 struct pci_dev *pdev = i915->drm.pdev; 978 unsigned int size; 979 u16 snb_gmch_ctl; 980 981 ggtt->gmadr = pci_resource(pdev, 2); 982 ggtt->mappable_end = resource_size(&ggtt->gmadr); 983 984 /* 985 * 64/512MB is the current min/max we actually know of, but this is 986 * just a coarse sanity check. 987 */ 988 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { 989 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 990 &ggtt->mappable_end); 991 return -ENXIO; 992 } 993 994 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 995 996 size = gen6_get_total_gtt_size(snb_gmch_ctl); 997 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 998 999 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1000 1001 ggtt->vm.clear_range = nop_clear_range; 1002 if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915)) 1003 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1004 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1005 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1006 ggtt->vm.cleanup = gen6_gmch_remove; 1007 1008 ggtt->invalidate = gen6_ggtt_invalidate; 1009 1010 if (HAS_EDRAM(i915)) 1011 ggtt->vm.pte_encode = iris_pte_encode; 1012 else if (IS_HASWELL(i915)) 1013 ggtt->vm.pte_encode = hsw_pte_encode; 1014 else if (IS_VALLEYVIEW(i915)) 1015 ggtt->vm.pte_encode = byt_pte_encode; 1016 else if (INTEL_GEN(i915) >= 7) 1017 ggtt->vm.pte_encode = ivb_pte_encode; 1018 else 1019 ggtt->vm.pte_encode = snb_pte_encode; 1020 1021 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1022 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1023 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1024 ggtt->vm.vma_ops.clear_pages = clear_pages; 1025 1026 return ggtt_probe_common(ggtt, size); 1027 } 1028 1029 static void i915_gmch_remove(struct i915_address_space *vm) 1030 { 1031 intel_gmch_remove(); 1032 } 1033 1034 static int i915_gmch_probe(struct i915_ggtt *ggtt) 1035 { 1036 struct drm_i915_private *i915 = ggtt->vm.i915; 1037 phys_addr_t gmadr_base; 1038 int ret; 1039 1040 ret = intel_gmch_probe(i915->bridge_dev, i915->drm.pdev, NULL); 1041 if (!ret) { 1042 drm_err(&i915->drm, "failed to set up gmch\n"); 1043 return -EIO; 1044 } 1045 1046 intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end); 1047 1048 ggtt->gmadr = 1049 (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end); 1050 1051 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1052 1053 ggtt->do_idle_maps = needs_idle_maps(i915); 1054 ggtt->vm.insert_page = i915_ggtt_insert_page; 1055 ggtt->vm.insert_entries = i915_ggtt_insert_entries; 1056 ggtt->vm.clear_range = i915_ggtt_clear_range; 1057 ggtt->vm.cleanup = i915_gmch_remove; 1058 1059 ggtt->invalidate = gmch_ggtt_invalidate; 1060 1061 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1062 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1063 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1064 ggtt->vm.vma_ops.clear_pages = clear_pages; 1065 1066 if (unlikely(ggtt->do_idle_maps)) 1067 drm_notice(&i915->drm, 1068 "Applying Ironlake quirks for intel_iommu\n"); 1069 1070 return 0; 1071 } 1072 1073 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1074 { 1075 struct drm_i915_private *i915 = gt->i915; 1076 int ret; 1077 1078 ggtt->vm.gt = gt; 1079 ggtt->vm.i915 = i915; 1080 ggtt->vm.dma = &i915->drm.pdev->dev; 1081 1082 if (INTEL_GEN(i915) <= 5) 1083 ret = i915_gmch_probe(ggtt); 1084 else if (INTEL_GEN(i915) < 8) 1085 ret = gen6_gmch_probe(ggtt); 1086 else 1087 ret = gen8_gmch_probe(ggtt); 1088 if (ret) 1089 return ret; 1090 1091 if ((ggtt->vm.total - 1) >> 32) { 1092 drm_err(&i915->drm, 1093 "We never expected a Global GTT with more than 32bits" 1094 " of address space! Found %lldM!\n", 1095 ggtt->vm.total >> 20); 1096 ggtt->vm.total = 1ULL << 32; 1097 ggtt->mappable_end = 1098 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1099 } 1100 1101 if (ggtt->mappable_end > ggtt->vm.total) { 1102 drm_err(&i915->drm, 1103 "mappable aperture extends past end of GGTT," 1104 " aperture=%pa, total=%llx\n", 1105 &ggtt->mappable_end, ggtt->vm.total); 1106 ggtt->mappable_end = ggtt->vm.total; 1107 } 1108 1109 /* GMADR is the PCI mmio aperture into the global GTT. */ 1110 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1111 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1112 (u64)ggtt->mappable_end >> 20); 1113 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1114 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1115 1116 return 0; 1117 } 1118 1119 /** 1120 * i915_ggtt_probe_hw - Probe GGTT hardware location 1121 * @i915: i915 device 1122 */ 1123 int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1124 { 1125 int ret; 1126 1127 ret = ggtt_probe_hw(&i915->ggtt, &i915->gt); 1128 if (ret) 1129 return ret; 1130 1131 if (intel_vtd_active()) 1132 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1133 1134 return 0; 1135 } 1136 1137 int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1138 { 1139 if (INTEL_GEN(i915) < 6 && !intel_enable_gtt()) 1140 return -EIO; 1141 1142 return 0; 1143 } 1144 1145 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt) 1146 { 1147 GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate); 1148 1149 ggtt->invalidate = guc_ggtt_invalidate; 1150 1151 ggtt->invalidate(ggtt); 1152 } 1153 1154 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt) 1155 { 1156 /* XXX Temporary pardon for error unload */ 1157 if (ggtt->invalidate == gen8_ggtt_invalidate) 1158 return; 1159 1160 /* We should only be called after i915_ggtt_enable_guc() */ 1161 GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate); 1162 1163 ggtt->invalidate = gen8_ggtt_invalidate; 1164 1165 ggtt->invalidate(ggtt); 1166 } 1167 1168 void i915_ggtt_resume(struct i915_ggtt *ggtt) 1169 { 1170 struct i915_vma *vma; 1171 bool flush = false; 1172 int open; 1173 1174 intel_gt_check_and_clear_faults(ggtt->vm.gt); 1175 1176 /* First fill our portion of the GTT with scratch pages */ 1177 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 1178 1179 /* Skip rewriting PTE on VMA unbind. */ 1180 open = atomic_xchg(&ggtt->vm.open, 0); 1181 1182 /* clflush objects bound into the GGTT and rebind them. */ 1183 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) { 1184 struct drm_i915_gem_object *obj = vma->obj; 1185 unsigned int was_bound = 1186 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1187 1188 GEM_BUG_ON(!was_bound); 1189 vma->ops->bind_vma(&ggtt->vm, NULL, vma, 1190 obj ? obj->cache_level : 0, 1191 was_bound); 1192 if (obj) { /* only used during resume => exclusive access */ 1193 flush |= fetch_and_zero(&obj->write_domain); 1194 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1195 } 1196 } 1197 1198 atomic_set(&ggtt->vm.open, open); 1199 ggtt->invalidate(ggtt); 1200 1201 if (flush) 1202 wbinvd_on_all_cpus(); 1203 1204 if (INTEL_GEN(ggtt->vm.i915) >= 8) 1205 setup_private_pat(ggtt->vm.gt->uncore); 1206 1207 intel_ggtt_restore_fences(ggtt); 1208 } 1209 1210 static struct scatterlist * 1211 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1212 unsigned int width, unsigned int height, 1213 unsigned int stride, 1214 struct sg_table *st, struct scatterlist *sg) 1215 { 1216 unsigned int column, row; 1217 unsigned int src_idx; 1218 1219 for (column = 0; column < width; column++) { 1220 src_idx = stride * (height - 1) + column + offset; 1221 for (row = 0; row < height; row++) { 1222 st->nents++; 1223 /* 1224 * We don't need the pages, but need to initialize 1225 * the entries so the sg list can be happily traversed. 1226 * The only thing we need are DMA addresses. 1227 */ 1228 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); 1229 sg_dma_address(sg) = 1230 i915_gem_object_get_dma_address(obj, src_idx); 1231 sg_dma_len(sg) = I915_GTT_PAGE_SIZE; 1232 sg = sg_next(sg); 1233 src_idx -= stride; 1234 } 1235 } 1236 1237 return sg; 1238 } 1239 1240 static noinline struct sg_table * 1241 intel_rotate_pages(struct intel_rotation_info *rot_info, 1242 struct drm_i915_gem_object *obj) 1243 { 1244 unsigned int size = intel_rotation_info_size(rot_info); 1245 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1246 struct sg_table *st; 1247 struct scatterlist *sg; 1248 int ret = -ENOMEM; 1249 int i; 1250 1251 /* Allocate target SG list. */ 1252 st = kmalloc(sizeof(*st), GFP_KERNEL); 1253 if (!st) 1254 goto err_st_alloc; 1255 1256 ret = sg_alloc_table(st, size, GFP_KERNEL); 1257 if (ret) 1258 goto err_sg_alloc; 1259 1260 st->nents = 0; 1261 sg = st->sgl; 1262 1263 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { 1264 sg = rotate_pages(obj, rot_info->plane[i].offset, 1265 rot_info->plane[i].width, rot_info->plane[i].height, 1266 rot_info->plane[i].stride, st, sg); 1267 } 1268 1269 return st; 1270 1271 err_sg_alloc: 1272 kfree(st); 1273 err_st_alloc: 1274 1275 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1276 obj->base.size, rot_info->plane[0].width, 1277 rot_info->plane[0].height, size); 1278 1279 return ERR_PTR(ret); 1280 } 1281 1282 static struct scatterlist * 1283 remap_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1284 unsigned int width, unsigned int height, 1285 unsigned int stride, 1286 struct sg_table *st, struct scatterlist *sg) 1287 { 1288 unsigned int row; 1289 1290 for (row = 0; row < height; row++) { 1291 unsigned int left = width * I915_GTT_PAGE_SIZE; 1292 1293 while (left) { 1294 dma_addr_t addr; 1295 unsigned int length; 1296 1297 /* 1298 * We don't need the pages, but need to initialize 1299 * the entries so the sg list can be happily traversed. 1300 * The only thing we need are DMA addresses. 1301 */ 1302 1303 addr = i915_gem_object_get_dma_address_len(obj, offset, &length); 1304 1305 length = min(left, length); 1306 1307 st->nents++; 1308 1309 sg_set_page(sg, NULL, length, 0); 1310 sg_dma_address(sg) = addr; 1311 sg_dma_len(sg) = length; 1312 sg = sg_next(sg); 1313 1314 offset += length / I915_GTT_PAGE_SIZE; 1315 left -= length; 1316 } 1317 1318 offset += stride - width; 1319 } 1320 1321 return sg; 1322 } 1323 1324 static noinline struct sg_table * 1325 intel_remap_pages(struct intel_remapped_info *rem_info, 1326 struct drm_i915_gem_object *obj) 1327 { 1328 unsigned int size = intel_remapped_info_size(rem_info); 1329 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1330 struct sg_table *st; 1331 struct scatterlist *sg; 1332 int ret = -ENOMEM; 1333 int i; 1334 1335 /* Allocate target SG list. */ 1336 st = kmalloc(sizeof(*st), GFP_KERNEL); 1337 if (!st) 1338 goto err_st_alloc; 1339 1340 ret = sg_alloc_table(st, size, GFP_KERNEL); 1341 if (ret) 1342 goto err_sg_alloc; 1343 1344 st->nents = 0; 1345 sg = st->sgl; 1346 1347 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { 1348 sg = remap_pages(obj, rem_info->plane[i].offset, 1349 rem_info->plane[i].width, rem_info->plane[i].height, 1350 rem_info->plane[i].stride, st, sg); 1351 } 1352 1353 i915_sg_trim(st); 1354 1355 return st; 1356 1357 err_sg_alloc: 1358 kfree(st); 1359 err_st_alloc: 1360 1361 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1362 obj->base.size, rem_info->plane[0].width, 1363 rem_info->plane[0].height, size); 1364 1365 return ERR_PTR(ret); 1366 } 1367 1368 static noinline struct sg_table * 1369 intel_partial_pages(const struct i915_ggtt_view *view, 1370 struct drm_i915_gem_object *obj) 1371 { 1372 struct sg_table *st; 1373 struct scatterlist *sg, *iter; 1374 unsigned int count = view->partial.size; 1375 unsigned int offset; 1376 int ret = -ENOMEM; 1377 1378 st = kmalloc(sizeof(*st), GFP_KERNEL); 1379 if (!st) 1380 goto err_st_alloc; 1381 1382 ret = sg_alloc_table(st, count, GFP_KERNEL); 1383 if (ret) 1384 goto err_sg_alloc; 1385 1386 iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); 1387 GEM_BUG_ON(!iter); 1388 1389 sg = st->sgl; 1390 st->nents = 0; 1391 do { 1392 unsigned int len; 1393 1394 len = min(iter->length - (offset << PAGE_SHIFT), 1395 count << PAGE_SHIFT); 1396 sg_set_page(sg, NULL, len, 0); 1397 sg_dma_address(sg) = 1398 sg_dma_address(iter) + (offset << PAGE_SHIFT); 1399 sg_dma_len(sg) = len; 1400 1401 st->nents++; 1402 count -= len >> PAGE_SHIFT; 1403 if (count == 0) { 1404 sg_mark_end(sg); 1405 i915_sg_trim(st); /* Drop any unused tail entries. */ 1406 1407 return st; 1408 } 1409 1410 sg = __sg_next(sg); 1411 iter = __sg_next(iter); 1412 offset = 0; 1413 } while (1); 1414 1415 err_sg_alloc: 1416 kfree(st); 1417 err_st_alloc: 1418 return ERR_PTR(ret); 1419 } 1420 1421 static int 1422 i915_get_ggtt_vma_pages(struct i915_vma *vma) 1423 { 1424 int ret; 1425 1426 /* 1427 * The vma->pages are only valid within the lifespan of the borrowed 1428 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 1429 * must be the vma->pages. A simple rule is that vma->pages must only 1430 * be accessed when the obj->mm.pages are pinned. 1431 */ 1432 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 1433 1434 switch (vma->ggtt_view.type) { 1435 default: 1436 GEM_BUG_ON(vma->ggtt_view.type); 1437 fallthrough; 1438 case I915_GGTT_VIEW_NORMAL: 1439 vma->pages = vma->obj->mm.pages; 1440 return 0; 1441 1442 case I915_GGTT_VIEW_ROTATED: 1443 vma->pages = 1444 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); 1445 break; 1446 1447 case I915_GGTT_VIEW_REMAPPED: 1448 vma->pages = 1449 intel_remap_pages(&vma->ggtt_view.remapped, vma->obj); 1450 break; 1451 1452 case I915_GGTT_VIEW_PARTIAL: 1453 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); 1454 break; 1455 } 1456 1457 ret = 0; 1458 if (IS_ERR(vma->pages)) { 1459 ret = PTR_ERR(vma->pages); 1460 vma->pages = NULL; 1461 drm_err(&vma->vm->i915->drm, 1462 "Failed to get pages for VMA view type %u (%d)!\n", 1463 vma->ggtt_view.type, ret); 1464 } 1465 return ret; 1466 } 1467