1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <asm/set_memory.h> 7 #include <asm/smp.h> 8 #include <linux/types.h> 9 #include <linux/stop_machine.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/i915_drm.h> 13 #include <drm/intel-gtt.h> 14 15 #include "gem/i915_gem_lmem.h" 16 17 #include "intel_ggtt_gmch.h" 18 #include "intel_gt.h" 19 #include "intel_gt_regs.h" 20 #include "intel_pci_config.h" 21 #include "i915_drv.h" 22 #include "i915_pci.h" 23 #include "i915_scatterlist.h" 24 #include "i915_utils.h" 25 #include "i915_vgpu.h" 26 27 #include "intel_gtt.h" 28 #include "gen8_ppgtt.h" 29 30 static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 31 unsigned long color, 32 u64 *start, 33 u64 *end) 34 { 35 if (i915_node_color_differs(node, color)) 36 *start += I915_GTT_PAGE_SIZE; 37 38 /* 39 * Also leave a space between the unallocated reserved node after the 40 * GTT and any objects within the GTT, i.e. we use the color adjustment 41 * to insert a guard page to prevent prefetches crossing over the 42 * GTT boundary. 43 */ 44 node = list_next_entry(node, node_list); 45 if (node->color != color) 46 *end -= I915_GTT_PAGE_SIZE; 47 } 48 49 static int ggtt_init_hw(struct i915_ggtt *ggtt) 50 { 51 struct drm_i915_private *i915 = ggtt->vm.i915; 52 53 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 54 55 ggtt->vm.is_ggtt = true; 56 57 /* Only VLV supports read-only GGTT mappings */ 58 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 59 60 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 61 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 62 63 if (ggtt->mappable_end) { 64 if (!io_mapping_init_wc(&ggtt->iomap, 65 ggtt->gmadr.start, 66 ggtt->mappable_end)) { 67 ggtt->vm.cleanup(&ggtt->vm); 68 return -EIO; 69 } 70 71 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 72 ggtt->mappable_end); 73 } 74 75 intel_ggtt_init_fences(ggtt); 76 77 return 0; 78 } 79 80 /** 81 * i915_ggtt_init_hw - Initialize GGTT hardware 82 * @i915: i915 device 83 */ 84 int i915_ggtt_init_hw(struct drm_i915_private *i915) 85 { 86 int ret; 87 88 /* 89 * Note that we use page colouring to enforce a guard page at the 90 * end of the address space. This is required as the CS may prefetch 91 * beyond the end of the batch buffer, across the page boundary, 92 * and beyond the end of the GTT if we do not provide a guard. 93 */ 94 ret = ggtt_init_hw(to_gt(i915)->ggtt); 95 if (ret) 96 return ret; 97 98 return 0; 99 } 100 101 /** 102 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM 103 * @vm: The VM to suspend the mappings for 104 * 105 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a 106 * DPT page table. 107 */ 108 void i915_ggtt_suspend_vm(struct i915_address_space *vm) 109 { 110 struct i915_vma *vma, *vn; 111 int save_skip_rewrite; 112 113 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 114 115 retry: 116 i915_gem_drain_freed_objects(vm->i915); 117 118 mutex_lock(&vm->mutex); 119 120 /* 121 * Skip rewriting PTE on VMA unbind. 122 * FIXME: Use an argument to i915_vma_unbind() instead? 123 */ 124 save_skip_rewrite = vm->skip_pte_rewrite; 125 vm->skip_pte_rewrite = true; 126 127 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 128 struct drm_i915_gem_object *obj = vma->obj; 129 130 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 131 132 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 133 continue; 134 135 /* unlikely to race when GPU is idle, so no worry about slowpath.. */ 136 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) { 137 /* 138 * No dead objects should appear here, GPU should be 139 * completely idle, and userspace suspended 140 */ 141 i915_gem_object_get(obj); 142 143 mutex_unlock(&vm->mutex); 144 145 i915_gem_object_lock(obj, NULL); 146 GEM_WARN_ON(i915_vma_unbind(vma)); 147 i915_gem_object_unlock(obj); 148 i915_gem_object_put(obj); 149 150 vm->skip_pte_rewrite = save_skip_rewrite; 151 goto retry; 152 } 153 154 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 155 i915_vma_wait_for_bind(vma); 156 157 __i915_vma_evict(vma, false); 158 drm_mm_remove_node(&vma->node); 159 } 160 161 i915_gem_object_unlock(obj); 162 } 163 164 vm->clear_range(vm, 0, vm->total); 165 166 vm->skip_pte_rewrite = save_skip_rewrite; 167 168 mutex_unlock(&vm->mutex); 169 } 170 171 void i915_ggtt_suspend(struct i915_ggtt *ggtt) 172 { 173 struct intel_gt *gt; 174 175 i915_ggtt_suspend_vm(&ggtt->vm); 176 ggtt->invalidate(ggtt); 177 178 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 179 intel_gt_check_and_clear_faults(gt); 180 } 181 182 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 183 { 184 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 185 186 spin_lock_irq(&uncore->lock); 187 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 188 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 189 spin_unlock_irq(&uncore->lock); 190 } 191 192 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 193 { 194 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 195 196 /* 197 * Note that as an uncached mmio write, this will flush the 198 * WCB of the writes into the GGTT before it triggers the invalidate. 199 */ 200 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 201 } 202 203 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 204 { 205 struct drm_i915_private *i915 = ggtt->vm.i915; 206 207 gen8_ggtt_invalidate(ggtt); 208 209 if (GRAPHICS_VER(i915) >= 12) { 210 struct intel_gt *gt; 211 212 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 213 intel_uncore_write_fw(gt->uncore, 214 GEN12_GUC_TLB_INV_CR, 215 GEN12_GUC_TLB_INV_CR_INVALIDATE); 216 } else { 217 intel_uncore_write_fw(ggtt->vm.gt->uncore, 218 GEN8_GTCR, GEN8_GTCR_INVALIDATE); 219 } 220 } 221 222 u64 gen8_ggtt_pte_encode(dma_addr_t addr, 223 enum i915_cache_level level, 224 u32 flags) 225 { 226 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 227 228 if (flags & PTE_LM) 229 pte |= GEN12_GGTT_PTE_LM; 230 231 return pte; 232 } 233 234 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 235 { 236 writeq(pte, addr); 237 } 238 239 static void gen8_ggtt_insert_page(struct i915_address_space *vm, 240 dma_addr_t addr, 241 u64 offset, 242 enum i915_cache_level level, 243 u32 flags) 244 { 245 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 246 gen8_pte_t __iomem *pte = 247 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 248 249 gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags)); 250 251 ggtt->invalidate(ggtt); 252 } 253 254 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 255 struct i915_vma_resource *vma_res, 256 enum i915_cache_level level, 257 u32 flags) 258 { 259 const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags); 260 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 261 gen8_pte_t __iomem *gte; 262 gen8_pte_t __iomem *end; 263 struct sgt_iter iter; 264 dma_addr_t addr; 265 266 /* 267 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 268 * not to allow the user to override access to a read only page. 269 */ 270 271 gte = (gen8_pte_t __iomem *)ggtt->gsm; 272 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 273 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 274 while (gte < end) 275 gen8_set_pte(gte++, vm->scratch[0]->encode); 276 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 277 278 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 279 gen8_set_pte(gte++, pte_encode | addr); 280 GEM_BUG_ON(gte > end); 281 282 /* Fill the allocated but "unused" space beyond the end of the buffer */ 283 while (gte < end) 284 gen8_set_pte(gte++, vm->scratch[0]->encode); 285 286 /* 287 * We want to flush the TLBs only after we're certain all the PTE 288 * updates have finished. 289 */ 290 ggtt->invalidate(ggtt); 291 } 292 293 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 294 u64 start, u64 length) 295 { 296 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 297 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 298 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 299 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 300 gen8_pte_t __iomem *gtt_base = 301 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 302 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 303 int i; 304 305 if (WARN(num_entries > max_entries, 306 "First entry = %d; Num entries = %d (max=%d)\n", 307 first_entry, num_entries, max_entries)) 308 num_entries = max_entries; 309 310 for (i = 0; i < num_entries; i++) 311 gen8_set_pte(>t_base[i], scratch_pte); 312 } 313 314 static void gen6_ggtt_insert_page(struct i915_address_space *vm, 315 dma_addr_t addr, 316 u64 offset, 317 enum i915_cache_level level, 318 u32 flags) 319 { 320 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 321 gen6_pte_t __iomem *pte = 322 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 323 324 iowrite32(vm->pte_encode(addr, level, flags), pte); 325 326 ggtt->invalidate(ggtt); 327 } 328 329 /* 330 * Binds an object into the global gtt with the specified cache level. 331 * The object will be accessible to the GPU via commands whose operands 332 * reference offsets within the global GTT as well as accessible by the GPU 333 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 334 */ 335 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 336 struct i915_vma_resource *vma_res, 337 enum i915_cache_level level, 338 u32 flags) 339 { 340 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 341 gen6_pte_t __iomem *gte; 342 gen6_pte_t __iomem *end; 343 struct sgt_iter iter; 344 dma_addr_t addr; 345 346 gte = (gen6_pte_t __iomem *)ggtt->gsm; 347 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 348 349 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 350 while (gte < end) 351 iowrite32(vm->scratch[0]->encode, gte++); 352 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 353 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 354 iowrite32(vm->pte_encode(addr, level, flags), gte++); 355 GEM_BUG_ON(gte > end); 356 357 /* Fill the allocated but "unused" space beyond the end of the buffer */ 358 while (gte < end) 359 iowrite32(vm->scratch[0]->encode, gte++); 360 361 /* 362 * We want to flush the TLBs only after we're certain all the PTE 363 * updates have finished. 364 */ 365 ggtt->invalidate(ggtt); 366 } 367 368 static void nop_clear_range(struct i915_address_space *vm, 369 u64 start, u64 length) 370 { 371 } 372 373 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 374 { 375 /* 376 * Make sure the internal GAM fifo has been cleared of all GTT 377 * writes before exiting stop_machine(). This guarantees that 378 * any aperture accesses waiting to start in another process 379 * cannot back up behind the GTT writes causing a hang. 380 * The register can be any arbitrary GAM register. 381 */ 382 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 383 } 384 385 struct insert_page { 386 struct i915_address_space *vm; 387 dma_addr_t addr; 388 u64 offset; 389 enum i915_cache_level level; 390 }; 391 392 static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 393 { 394 struct insert_page *arg = _arg; 395 396 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 397 bxt_vtd_ggtt_wa(arg->vm); 398 399 return 0; 400 } 401 402 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 403 dma_addr_t addr, 404 u64 offset, 405 enum i915_cache_level level, 406 u32 unused) 407 { 408 struct insert_page arg = { vm, addr, offset, level }; 409 410 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 411 } 412 413 struct insert_entries { 414 struct i915_address_space *vm; 415 struct i915_vma_resource *vma_res; 416 enum i915_cache_level level; 417 u32 flags; 418 }; 419 420 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 421 { 422 struct insert_entries *arg = _arg; 423 424 gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags); 425 bxt_vtd_ggtt_wa(arg->vm); 426 427 return 0; 428 } 429 430 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 431 struct i915_vma_resource *vma_res, 432 enum i915_cache_level level, 433 u32 flags) 434 { 435 struct insert_entries arg = { vm, vma_res, level, flags }; 436 437 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 438 } 439 440 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 441 u64 start, u64 length) 442 { 443 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 444 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 445 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 446 gen6_pte_t scratch_pte, __iomem *gtt_base = 447 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 448 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 449 int i; 450 451 if (WARN(num_entries > max_entries, 452 "First entry = %d; Num entries = %d (max=%d)\n", 453 first_entry, num_entries, max_entries)) 454 num_entries = max_entries; 455 456 scratch_pte = vm->scratch[0]->encode; 457 for (i = 0; i < num_entries; i++) 458 iowrite32(scratch_pte, >t_base[i]); 459 } 460 461 void intel_ggtt_bind_vma(struct i915_address_space *vm, 462 struct i915_vm_pt_stash *stash, 463 struct i915_vma_resource *vma_res, 464 enum i915_cache_level cache_level, 465 u32 flags) 466 { 467 u32 pte_flags; 468 469 if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK)) 470 return; 471 472 vma_res->bound_flags |= flags; 473 474 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 475 pte_flags = 0; 476 if (vma_res->bi.readonly) 477 pte_flags |= PTE_READ_ONLY; 478 if (vma_res->bi.lmem) 479 pte_flags |= PTE_LM; 480 481 vm->insert_entries(vm, vma_res, cache_level, pte_flags); 482 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 483 } 484 485 void intel_ggtt_unbind_vma(struct i915_address_space *vm, 486 struct i915_vma_resource *vma_res) 487 { 488 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 489 } 490 491 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 492 { 493 u64 size; 494 int ret; 495 496 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 497 return 0; 498 499 GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); 500 size = ggtt->vm.total - GUC_GGTT_TOP; 501 502 ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size, 503 GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, 504 PIN_NOEVICT); 505 if (ret) 506 drm_dbg(&ggtt->vm.i915->drm, 507 "Failed to reserve top of GGTT for GuC\n"); 508 509 return ret; 510 } 511 512 static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 513 { 514 if (drm_mm_node_allocated(&ggtt->uc_fw)) 515 drm_mm_remove_node(&ggtt->uc_fw); 516 } 517 518 static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 519 { 520 ggtt_release_guc_top(ggtt); 521 if (drm_mm_node_allocated(&ggtt->error_capture)) 522 drm_mm_remove_node(&ggtt->error_capture); 523 mutex_destroy(&ggtt->error_mutex); 524 } 525 526 static int init_ggtt(struct i915_ggtt *ggtt) 527 { 528 /* 529 * Let GEM Manage all of the aperture. 530 * 531 * However, leave one page at the end still bound to the scratch page. 532 * There are a number of places where the hardware apparently prefetches 533 * past the end of the object, and we've seen multiple hangs with the 534 * GPU head pointer stuck in a batchbuffer bound at the last page of the 535 * aperture. One page should be enough to keep any prefetching inside 536 * of the aperture. 537 */ 538 unsigned long hole_start, hole_end; 539 struct drm_mm_node *entry; 540 int ret; 541 542 /* 543 * GuC requires all resources that we're sharing with it to be placed in 544 * non-WOPCM memory. If GuC is not present or not in use we still need a 545 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 546 * why. 547 */ 548 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 549 intel_wopcm_guc_size(&ggtt->vm.gt->wopcm)); 550 551 ret = intel_vgt_balloon(ggtt); 552 if (ret) 553 return ret; 554 555 mutex_init(&ggtt->error_mutex); 556 if (ggtt->mappable_end) { 557 /* 558 * Reserve a mappable slot for our lockless error capture. 559 * 560 * We strongly prefer taking address 0x0 in order to protect 561 * other critical buffers against accidental overwrites, 562 * as writing to address 0 is a very common mistake. 563 * 564 * Since 0 may already be in use by the system (e.g. the BIOS 565 * framebuffer), we let the reservation fail quietly and hope 566 * 0 remains reserved always. 567 * 568 * If we fail to reserve 0, and then fail to find any space 569 * for an error-capture, remain silent. We can afford not 570 * to reserve an error_capture node as we have fallback 571 * paths, and we trust that 0 will remain reserved. However, 572 * the only likely reason for failure to insert is a driver 573 * bug, which we expect to cause other failures... 574 */ 575 ggtt->error_capture.size = I915_GTT_PAGE_SIZE; 576 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 577 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 578 drm_mm_insert_node_in_range(&ggtt->vm.mm, 579 &ggtt->error_capture, 580 ggtt->error_capture.size, 0, 581 ggtt->error_capture.color, 582 0, ggtt->mappable_end, 583 DRM_MM_INSERT_LOW); 584 } 585 if (drm_mm_node_allocated(&ggtt->error_capture)) 586 drm_dbg(&ggtt->vm.i915->drm, 587 "Reserved GGTT:[%llx, %llx] for use by error capture\n", 588 ggtt->error_capture.start, 589 ggtt->error_capture.start + ggtt->error_capture.size); 590 591 /* 592 * The upper portion of the GuC address space has a sizeable hole 593 * (several MB) that is inaccessible by GuC. Reserve this range within 594 * GGTT as it can comfortably hold GuC/HuC firmware images. 595 */ 596 ret = ggtt_reserve_guc_top(ggtt); 597 if (ret) 598 goto err; 599 600 /* Clear any non-preallocated blocks */ 601 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 602 drm_dbg(&ggtt->vm.i915->drm, 603 "clearing unused GTT space: [%lx, %lx]\n", 604 hole_start, hole_end); 605 ggtt->vm.clear_range(&ggtt->vm, hole_start, 606 hole_end - hole_start); 607 } 608 609 /* And finally clear the reserved guard page */ 610 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 611 612 return 0; 613 614 err: 615 cleanup_init_ggtt(ggtt); 616 return ret; 617 } 618 619 static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 620 struct i915_vm_pt_stash *stash, 621 struct i915_vma_resource *vma_res, 622 enum i915_cache_level cache_level, 623 u32 flags) 624 { 625 u32 pte_flags; 626 627 /* Currently applicable only to VLV */ 628 pte_flags = 0; 629 if (vma_res->bi.readonly) 630 pte_flags |= PTE_READ_ONLY; 631 632 if (flags & I915_VMA_LOCAL_BIND) 633 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 634 stash, vma_res, cache_level, flags); 635 636 if (flags & I915_VMA_GLOBAL_BIND) 637 vm->insert_entries(vm, vma_res, cache_level, pte_flags); 638 639 vma_res->bound_flags |= flags; 640 } 641 642 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 643 struct i915_vma_resource *vma_res) 644 { 645 if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND) 646 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 647 648 if (vma_res->bound_flags & I915_VMA_LOCAL_BIND) 649 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res); 650 } 651 652 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 653 { 654 struct i915_vm_pt_stash stash = {}; 655 struct i915_ppgtt *ppgtt; 656 int err; 657 658 ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0); 659 if (IS_ERR(ppgtt)) 660 return PTR_ERR(ppgtt); 661 662 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 663 err = -ENODEV; 664 goto err_ppgtt; 665 } 666 667 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 668 if (err) 669 goto err_ppgtt; 670 671 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); 672 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 673 i915_gem_object_unlock(ppgtt->vm.scratch[0]); 674 if (err) 675 goto err_stash; 676 677 /* 678 * Note we only pre-allocate as far as the end of the global 679 * GTT. On 48b / 4-level page-tables, the difference is very, 680 * very significant! We have to preallocate as GVT/vgpu does 681 * not like the page directory disappearing. 682 */ 683 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 684 685 ggtt->alias = ppgtt; 686 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 687 688 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma); 689 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 690 691 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma); 692 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 693 694 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 695 return 0; 696 697 err_stash: 698 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 699 err_ppgtt: 700 i915_vm_put(&ppgtt->vm); 701 return err; 702 } 703 704 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 705 { 706 struct i915_ppgtt *ppgtt; 707 708 ppgtt = fetch_and_zero(&ggtt->alias); 709 if (!ppgtt) 710 return; 711 712 i915_vm_put(&ppgtt->vm); 713 714 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 715 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 716 } 717 718 int i915_init_ggtt(struct drm_i915_private *i915) 719 { 720 int ret; 721 722 ret = init_ggtt(to_gt(i915)->ggtt); 723 if (ret) 724 return ret; 725 726 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 727 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); 728 if (ret) 729 cleanup_init_ggtt(to_gt(i915)->ggtt); 730 } 731 732 return 0; 733 } 734 735 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 736 { 737 struct i915_vma *vma, *vn; 738 739 flush_workqueue(ggtt->vm.i915->wq); 740 i915_gem_drain_freed_objects(ggtt->vm.i915); 741 742 mutex_lock(&ggtt->vm.mutex); 743 744 ggtt->vm.skip_pte_rewrite = true; 745 746 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 747 struct drm_i915_gem_object *obj = vma->obj; 748 bool trylock; 749 750 trylock = i915_gem_object_trylock(obj, NULL); 751 WARN_ON(!trylock); 752 753 WARN_ON(__i915_vma_unbind(vma)); 754 if (trylock) 755 i915_gem_object_unlock(obj); 756 } 757 758 if (drm_mm_node_allocated(&ggtt->error_capture)) 759 drm_mm_remove_node(&ggtt->error_capture); 760 mutex_destroy(&ggtt->error_mutex); 761 762 ggtt_release_guc_top(ggtt); 763 intel_vgt_deballoon(ggtt); 764 765 ggtt->vm.cleanup(&ggtt->vm); 766 767 mutex_unlock(&ggtt->vm.mutex); 768 i915_address_space_fini(&ggtt->vm); 769 770 arch_phys_wc_del(ggtt->mtrr); 771 772 if (ggtt->iomap.size) 773 io_mapping_fini(&ggtt->iomap); 774 } 775 776 /** 777 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 778 * @i915: i915 device 779 */ 780 void i915_ggtt_driver_release(struct drm_i915_private *i915) 781 { 782 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 783 784 fini_aliasing_ppgtt(ggtt); 785 786 intel_ggtt_fini_fences(ggtt); 787 ggtt_cleanup_hw(ggtt); 788 } 789 790 /** 791 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after 792 * all free objects have been drained. 793 * @i915: i915 device 794 */ 795 void i915_ggtt_driver_late_release(struct drm_i915_private *i915) 796 { 797 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 798 799 GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); 800 dma_resv_fini(&ggtt->vm._resv); 801 } 802 803 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 804 { 805 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 806 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 807 return snb_gmch_ctl << 20; 808 } 809 810 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 811 { 812 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 813 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 814 if (bdw_gmch_ctl) 815 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 816 817 #ifdef CONFIG_X86_32 818 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 819 if (bdw_gmch_ctl > 4) 820 bdw_gmch_ctl = 4; 821 #endif 822 823 return bdw_gmch_ctl << 20; 824 } 825 826 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 827 { 828 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 829 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 830 831 if (gmch_ctrl) 832 return 1 << (20 + gmch_ctrl); 833 834 return 0; 835 } 836 837 static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) 838 { 839 /* 840 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset 841 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset 842 */ 843 GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 844 return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M; 845 } 846 847 static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) 848 { 849 return gen6_gttmmadr_size(i915) / 2; 850 } 851 852 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 853 { 854 struct drm_i915_private *i915 = ggtt->vm.i915; 855 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 856 phys_addr_t phys_addr; 857 u32 pte_flags; 858 int ret; 859 860 GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915)); 861 phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915); 862 863 /* 864 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range 865 * will be dropped. For WC mappings in general we have 64 byte burst 866 * writes when the WC buffer is flushed, so we can't use it, but have to 867 * resort to an uncached mapping. The WC issue is easily caught by the 868 * readback check when writing GTT PTE entries. 869 */ 870 if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 11) 871 ggtt->gsm = ioremap(phys_addr, size); 872 else 873 ggtt->gsm = ioremap_wc(phys_addr, size); 874 if (!ggtt->gsm) { 875 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 876 return -ENOMEM; 877 } 878 879 kref_init(&ggtt->vm.resv_ref); 880 ret = setup_scratch_page(&ggtt->vm); 881 if (ret) { 882 drm_err(&i915->drm, "Scratch setup failed\n"); 883 /* iounmap will also get called at remove, but meh */ 884 iounmap(ggtt->gsm); 885 return ret; 886 } 887 888 pte_flags = 0; 889 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0])) 890 pte_flags |= PTE_LM; 891 892 ggtt->vm.scratch[0]->encode = 893 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 894 I915_CACHE_NONE, pte_flags); 895 896 return 0; 897 } 898 899 static void gen6_gmch_remove(struct i915_address_space *vm) 900 { 901 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 902 903 iounmap(ggtt->gsm); 904 free_scratch(vm); 905 } 906 907 static struct resource pci_resource(struct pci_dev *pdev, int bar) 908 { 909 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar), 910 pci_resource_len(pdev, bar)); 911 } 912 913 static int gen8_gmch_probe(struct i915_ggtt *ggtt) 914 { 915 struct drm_i915_private *i915 = ggtt->vm.i915; 916 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 917 unsigned int size; 918 u16 snb_gmch_ctl; 919 920 if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) { 921 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 922 return -ENXIO; 923 924 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 925 ggtt->mappable_end = resource_size(&ggtt->gmadr); 926 } 927 928 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 929 if (IS_CHERRYVIEW(i915)) 930 size = chv_get_total_gtt_size(snb_gmch_ctl); 931 else 932 size = gen8_get_total_gtt_size(snb_gmch_ctl); 933 934 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 935 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 936 ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY; 937 938 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 939 ggtt->vm.cleanup = gen6_gmch_remove; 940 ggtt->vm.insert_page = gen8_ggtt_insert_page; 941 ggtt->vm.clear_range = nop_clear_range; 942 ggtt->vm.scratch_range = gen8_ggtt_clear_range; 943 944 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 945 946 /* 947 * Serialize GTT updates with aperture access on BXT if VT-d is on, 948 * and always on CHV. 949 */ 950 if (intel_vm_no_concurrent_access_wa(i915)) { 951 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 952 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 953 954 /* 955 * Calling stop_machine() version of GGTT update function 956 * at error capture/reset path will raise lockdep warning. 957 * Allow calling gen8_ggtt_insert_* directly at reset path 958 * which is safe from parallel GGTT updates. 959 */ 960 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; 961 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries; 962 963 ggtt->vm.bind_async_flags = 964 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 965 } 966 967 if (intel_uc_wants_guc(&ggtt->vm.gt->uc)) 968 ggtt->invalidate = guc_ggtt_invalidate; 969 else 970 ggtt->invalidate = gen8_ggtt_invalidate; 971 972 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 973 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 974 975 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 976 977 return ggtt_probe_common(ggtt, size); 978 } 979 980 static u64 snb_pte_encode(dma_addr_t addr, 981 enum i915_cache_level level, 982 u32 flags) 983 { 984 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 985 986 switch (level) { 987 case I915_CACHE_L3_LLC: 988 case I915_CACHE_LLC: 989 pte |= GEN6_PTE_CACHE_LLC; 990 break; 991 case I915_CACHE_NONE: 992 pte |= GEN6_PTE_UNCACHED; 993 break; 994 default: 995 MISSING_CASE(level); 996 } 997 998 return pte; 999 } 1000 1001 static u64 ivb_pte_encode(dma_addr_t addr, 1002 enum i915_cache_level level, 1003 u32 flags) 1004 { 1005 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1006 1007 switch (level) { 1008 case I915_CACHE_L3_LLC: 1009 pte |= GEN7_PTE_CACHE_L3_LLC; 1010 break; 1011 case I915_CACHE_LLC: 1012 pte |= GEN6_PTE_CACHE_LLC; 1013 break; 1014 case I915_CACHE_NONE: 1015 pte |= GEN6_PTE_UNCACHED; 1016 break; 1017 default: 1018 MISSING_CASE(level); 1019 } 1020 1021 return pte; 1022 } 1023 1024 static u64 byt_pte_encode(dma_addr_t addr, 1025 enum i915_cache_level level, 1026 u32 flags) 1027 { 1028 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1029 1030 if (!(flags & PTE_READ_ONLY)) 1031 pte |= BYT_PTE_WRITEABLE; 1032 1033 if (level != I915_CACHE_NONE) 1034 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 1035 1036 return pte; 1037 } 1038 1039 static u64 hsw_pte_encode(dma_addr_t addr, 1040 enum i915_cache_level level, 1041 u32 flags) 1042 { 1043 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1044 1045 if (level != I915_CACHE_NONE) 1046 pte |= HSW_WB_LLC_AGE3; 1047 1048 return pte; 1049 } 1050 1051 static u64 iris_pte_encode(dma_addr_t addr, 1052 enum i915_cache_level level, 1053 u32 flags) 1054 { 1055 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1056 1057 switch (level) { 1058 case I915_CACHE_NONE: 1059 break; 1060 case I915_CACHE_WT: 1061 pte |= HSW_WT_ELLC_LLC_AGE3; 1062 break; 1063 default: 1064 pte |= HSW_WB_ELLC_LLC_AGE3; 1065 break; 1066 } 1067 1068 return pte; 1069 } 1070 1071 static int gen6_gmch_probe(struct i915_ggtt *ggtt) 1072 { 1073 struct drm_i915_private *i915 = ggtt->vm.i915; 1074 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1075 unsigned int size; 1076 u16 snb_gmch_ctl; 1077 1078 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 1079 return -ENXIO; 1080 1081 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 1082 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1083 1084 /* 1085 * 64/512MB is the current min/max we actually know of, but this is 1086 * just a coarse sanity check. 1087 */ 1088 if (ggtt->mappable_end < (64 << 20) || 1089 ggtt->mappable_end > (512 << 20)) { 1090 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 1091 &ggtt->mappable_end); 1092 return -ENXIO; 1093 } 1094 1095 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1096 1097 size = gen6_get_total_gtt_size(snb_gmch_ctl); 1098 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 1099 1100 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1101 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 1102 1103 ggtt->vm.clear_range = nop_clear_range; 1104 if (!HAS_FULL_PPGTT(i915)) 1105 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1106 ggtt->vm.scratch_range = gen6_ggtt_clear_range; 1107 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1108 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1109 ggtt->vm.cleanup = gen6_gmch_remove; 1110 1111 ggtt->invalidate = gen6_ggtt_invalidate; 1112 1113 if (HAS_EDRAM(i915)) 1114 ggtt->vm.pte_encode = iris_pte_encode; 1115 else if (IS_HASWELL(i915)) 1116 ggtt->vm.pte_encode = hsw_pte_encode; 1117 else if (IS_VALLEYVIEW(i915)) 1118 ggtt->vm.pte_encode = byt_pte_encode; 1119 else if (GRAPHICS_VER(i915) >= 7) 1120 ggtt->vm.pte_encode = ivb_pte_encode; 1121 else 1122 ggtt->vm.pte_encode = snb_pte_encode; 1123 1124 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1125 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1126 1127 return ggtt_probe_common(ggtt, size); 1128 } 1129 1130 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1131 { 1132 struct drm_i915_private *i915 = gt->i915; 1133 int ret; 1134 1135 ggtt->vm.gt = gt; 1136 ggtt->vm.i915 = i915; 1137 ggtt->vm.dma = i915->drm.dev; 1138 dma_resv_init(&ggtt->vm._resv); 1139 1140 if (GRAPHICS_VER(i915) >= 8) 1141 ret = gen8_gmch_probe(ggtt); 1142 else if (GRAPHICS_VER(i915) >= 6) 1143 ret = gen6_gmch_probe(ggtt); 1144 else 1145 ret = intel_ggtt_gmch_probe(ggtt); 1146 1147 if (ret) { 1148 dma_resv_fini(&ggtt->vm._resv); 1149 return ret; 1150 } 1151 1152 if ((ggtt->vm.total - 1) >> 32) { 1153 drm_err(&i915->drm, 1154 "We never expected a Global GTT with more than 32bits" 1155 " of address space! Found %lldM!\n", 1156 ggtt->vm.total >> 20); 1157 ggtt->vm.total = 1ULL << 32; 1158 ggtt->mappable_end = 1159 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1160 } 1161 1162 if (ggtt->mappable_end > ggtt->vm.total) { 1163 drm_err(&i915->drm, 1164 "mappable aperture extends past end of GGTT," 1165 " aperture=%pa, total=%llx\n", 1166 &ggtt->mappable_end, ggtt->vm.total); 1167 ggtt->mappable_end = ggtt->vm.total; 1168 } 1169 1170 /* GMADR is the PCI mmio aperture into the global GTT. */ 1171 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1172 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1173 (u64)ggtt->mappable_end >> 20); 1174 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1175 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1176 1177 return 0; 1178 } 1179 1180 /** 1181 * i915_ggtt_probe_hw - Probe GGTT hardware location 1182 * @i915: i915 device 1183 */ 1184 int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1185 { 1186 struct intel_gt *gt; 1187 int ret, i; 1188 1189 for_each_gt(gt, i915, i) { 1190 ret = intel_gt_assign_ggtt(gt); 1191 if (ret) 1192 return ret; 1193 } 1194 1195 ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); 1196 if (ret) 1197 return ret; 1198 1199 if (i915_vtd_active(i915)) 1200 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1201 1202 return 0; 1203 } 1204 1205 struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915) 1206 { 1207 struct i915_ggtt *ggtt; 1208 1209 ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL); 1210 if (!ggtt) 1211 return ERR_PTR(-ENOMEM); 1212 1213 INIT_LIST_HEAD(&ggtt->gt_list); 1214 1215 return ggtt; 1216 } 1217 1218 int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1219 { 1220 if (GRAPHICS_VER(i915) < 6) 1221 return intel_ggtt_gmch_enable_hw(i915); 1222 1223 return 0; 1224 } 1225 1226 /** 1227 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM 1228 * @vm: The VM to restore the mappings for 1229 * 1230 * Restore the memory mappings for all objects mapped to HW via the GGTT or a 1231 * DPT page table. 1232 * 1233 * Returns %true if restoring the mapping for any object that was in a write 1234 * domain before suspend. 1235 */ 1236 bool i915_ggtt_resume_vm(struct i915_address_space *vm) 1237 { 1238 struct i915_vma *vma; 1239 bool write_domain_objs = false; 1240 1241 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 1242 1243 /* First fill our portion of the GTT with scratch pages */ 1244 vm->clear_range(vm, 0, vm->total); 1245 1246 /* clflush objects bound into the GGTT and rebind them. */ 1247 list_for_each_entry(vma, &vm->bound_list, vm_link) { 1248 struct drm_i915_gem_object *obj = vma->obj; 1249 unsigned int was_bound = 1250 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1251 1252 GEM_BUG_ON(!was_bound); 1253 1254 /* 1255 * Clear the bound flags of the vma resource to allow 1256 * ptes to be repopulated. 1257 */ 1258 vma->resource->bound_flags = 0; 1259 vma->ops->bind_vma(vm, NULL, vma->resource, 1260 obj ? obj->cache_level : 0, 1261 was_bound); 1262 1263 if (obj) { /* only used during resume => exclusive access */ 1264 write_domain_objs |= fetch_and_zero(&obj->write_domain); 1265 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1266 } 1267 } 1268 1269 return write_domain_objs; 1270 } 1271 1272 void i915_ggtt_resume(struct i915_ggtt *ggtt) 1273 { 1274 struct intel_gt *gt; 1275 bool flush; 1276 1277 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1278 intel_gt_check_and_clear_faults(gt); 1279 1280 flush = i915_ggtt_resume_vm(&ggtt->vm); 1281 1282 ggtt->invalidate(ggtt); 1283 1284 if (flush) 1285 wbinvd_on_all_cpus(); 1286 1287 intel_ggtt_restore_fences(ggtt); 1288 } 1289