1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "i915_gem_object.h" 9 #include "i915_scatterlist.h" 10 #include "i915_gem_lmem.h" 11 #include "i915_gem_mman.h" 12 13 #include "gt/intel_gt.h" 14 15 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 16 struct sg_table *pages, 17 unsigned int sg_page_sizes) 18 { 19 struct drm_i915_private *i915 = to_i915(obj->base.dev); 20 unsigned long supported = INTEL_INFO(i915)->page_sizes; 21 bool shrinkable; 22 int i; 23 24 assert_object_held_shared(obj); 25 26 if (i915_gem_object_is_volatile(obj)) 27 obj->mm.madv = I915_MADV_DONTNEED; 28 29 /* Make the pages coherent with the GPU (flushing any swapin). */ 30 if (obj->cache_dirty) { 31 WARN_ON_ONCE(IS_DGFX(i915)); 32 obj->write_domain = 0; 33 if (i915_gem_object_has_struct_page(obj)) 34 drm_clflush_sg(pages); 35 obj->cache_dirty = false; 36 } 37 38 obj->mm.get_page.sg_pos = pages->sgl; 39 obj->mm.get_page.sg_idx = 0; 40 obj->mm.get_dma_page.sg_pos = pages->sgl; 41 obj->mm.get_dma_page.sg_idx = 0; 42 43 obj->mm.pages = pages; 44 45 GEM_BUG_ON(!sg_page_sizes); 46 obj->mm.page_sizes.phys = sg_page_sizes; 47 48 /* 49 * Calculate the supported page-sizes which fit into the given 50 * sg_page_sizes. This will give us the page-sizes which we may be able 51 * to use opportunistically when later inserting into the GTT. For 52 * example if phys=2G, then in theory we should be able to use 1G, 2M, 53 * 64K or 4K pages, although in practice this will depend on a number of 54 * other factors. 55 */ 56 obj->mm.page_sizes.sg = 0; 57 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 58 if (obj->mm.page_sizes.phys & ~0u << i) 59 obj->mm.page_sizes.sg |= BIT(i); 60 } 61 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); 62 63 shrinkable = i915_gem_object_is_shrinkable(obj); 64 65 if (i915_gem_object_is_tiled(obj) && 66 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 67 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); 68 i915_gem_object_set_tiling_quirk(obj); 69 GEM_BUG_ON(!list_empty(&obj->mm.link)); 70 atomic_inc(&obj->mm.shrink_pin); 71 shrinkable = false; 72 } 73 74 if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) { 75 struct list_head *list; 76 unsigned long flags; 77 78 assert_object_held(obj); 79 spin_lock_irqsave(&i915->mm.obj_lock, flags); 80 81 i915->mm.shrink_count++; 82 i915->mm.shrink_memory += obj->base.size; 83 84 if (obj->mm.madv != I915_MADV_WILLNEED) 85 list = &i915->mm.purge_list; 86 else 87 list = &i915->mm.shrink_list; 88 list_add_tail(&obj->mm.link, list); 89 90 atomic_set(&obj->mm.shrink_pin, 0); 91 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 92 } 93 } 94 95 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 96 { 97 struct drm_i915_private *i915 = to_i915(obj->base.dev); 98 int err; 99 100 assert_object_held_shared(obj); 101 102 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 103 drm_dbg(&i915->drm, 104 "Attempting to obtain a purgeable object\n"); 105 return -EFAULT; 106 } 107 108 err = obj->ops->get_pages(obj); 109 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); 110 111 return err; 112 } 113 114 /* Ensure that the associated pages are gathered from the backing storage 115 * and pinned into our object. i915_gem_object_pin_pages() may be called 116 * multiple times before they are released by a single call to 117 * i915_gem_object_unpin_pages() - once the pages are no longer referenced 118 * either as a result of memory pressure (reaping pages under the shrinker) 119 * or as the object is itself released. 120 */ 121 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 122 { 123 int err; 124 125 assert_object_held(obj); 126 127 assert_object_held_shared(obj); 128 129 if (unlikely(!i915_gem_object_has_pages(obj))) { 130 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 131 132 err = ____i915_gem_object_get_pages(obj); 133 if (err) 134 return err; 135 136 smp_mb__before_atomic(); 137 } 138 atomic_inc(&obj->mm.pages_pin_count); 139 140 return 0; 141 } 142 143 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj) 144 { 145 struct i915_gem_ww_ctx ww; 146 int err; 147 148 i915_gem_ww_ctx_init(&ww, true); 149 retry: 150 err = i915_gem_object_lock(obj, &ww); 151 if (!err) 152 err = i915_gem_object_pin_pages(obj); 153 154 if (err == -EDEADLK) { 155 err = i915_gem_ww_ctx_backoff(&ww); 156 if (!err) 157 goto retry; 158 } 159 i915_gem_ww_ctx_fini(&ww); 160 return err; 161 } 162 163 /* Immediately discard the backing storage */ 164 int i915_gem_object_truncate(struct drm_i915_gem_object *obj) 165 { 166 if (obj->ops->truncate) 167 return obj->ops->truncate(obj); 168 169 return 0; 170 } 171 172 /* Try to discard unwanted pages */ 173 void i915_gem_object_writeback(struct drm_i915_gem_object *obj) 174 { 175 assert_object_held_shared(obj); 176 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 177 178 if (obj->ops->writeback) 179 obj->ops->writeback(obj); 180 } 181 182 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) 183 { 184 struct radix_tree_iter iter; 185 void __rcu **slot; 186 187 rcu_read_lock(); 188 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) 189 radix_tree_delete(&obj->mm.get_page.radix, iter.index); 190 radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0) 191 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index); 192 rcu_read_unlock(); 193 } 194 195 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr) 196 { 197 if (is_vmalloc_addr(ptr)) 198 vunmap(ptr); 199 } 200 201 struct sg_table * 202 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) 203 { 204 struct sg_table *pages; 205 206 assert_object_held_shared(obj); 207 208 pages = fetch_and_zero(&obj->mm.pages); 209 if (IS_ERR_OR_NULL(pages)) 210 return pages; 211 212 if (i915_gem_object_is_volatile(obj)) 213 obj->mm.madv = I915_MADV_WILLNEED; 214 215 if (!i915_gem_object_has_self_managed_shrink_list(obj)) 216 i915_gem_object_make_unshrinkable(obj); 217 218 if (obj->mm.mapping) { 219 unmap_object(obj, page_mask_bits(obj->mm.mapping)); 220 obj->mm.mapping = NULL; 221 } 222 223 __i915_gem_object_reset_page_iter(obj); 224 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; 225 226 if (test_and_clear_bit(I915_BO_WAS_BOUND_BIT, &obj->flags)) { 227 struct drm_i915_private *i915 = to_i915(obj->base.dev); 228 intel_wakeref_t wakeref; 229 230 with_intel_runtime_pm_if_active(&i915->runtime_pm, wakeref) 231 intel_gt_invalidate_tlbs(to_gt(i915)); 232 } 233 234 return pages; 235 } 236 237 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) 238 { 239 struct sg_table *pages; 240 241 if (i915_gem_object_has_pinned_pages(obj)) 242 return -EBUSY; 243 244 /* May be called by shrinker from within get_pages() (on another bo) */ 245 assert_object_held_shared(obj); 246 247 i915_gem_object_release_mmap_offset(obj); 248 249 /* 250 * ->put_pages might need to allocate memory for the bit17 swizzle 251 * array, hence protect them from being reaped by removing them from gtt 252 * lists early. 253 */ 254 pages = __i915_gem_object_unset_pages(obj); 255 256 /* 257 * XXX Temporary hijinx to avoid updating all backends to handle 258 * NULL pages. In the future, when we have more asynchronous 259 * get_pages backends we should be better able to handle the 260 * cancellation of the async task in a more uniform manner. 261 */ 262 if (!IS_ERR_OR_NULL(pages)) 263 obj->ops->put_pages(obj, pages); 264 265 return 0; 266 } 267 268 /* The 'mapping' part of i915_gem_object_pin_map() below */ 269 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj, 270 enum i915_map_type type) 271 { 272 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i; 273 struct page *stack[32], **pages = stack, *page; 274 struct sgt_iter iter; 275 pgprot_t pgprot; 276 void *vaddr; 277 278 switch (type) { 279 default: 280 MISSING_CASE(type); 281 fallthrough; /* to use PAGE_KERNEL anyway */ 282 case I915_MAP_WB: 283 /* 284 * On 32b, highmem using a finite set of indirect PTE (i.e. 285 * vmap) to provide virtual mappings of the high pages. 286 * As these are finite, map_new_virtual() must wait for some 287 * other kmap() to finish when it runs out. If we map a large 288 * number of objects, there is no method for it to tell us 289 * to release the mappings, and we deadlock. 290 * 291 * However, if we make an explicit vmap of the page, that 292 * uses a larger vmalloc arena, and also has the ability 293 * to tell us to release unwanted mappings. Most importantly, 294 * it will fail and propagate an error instead of waiting 295 * forever. 296 * 297 * So if the page is beyond the 32b boundary, make an explicit 298 * vmap. 299 */ 300 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))) 301 return page_address(sg_page(obj->mm.pages->sgl)); 302 pgprot = PAGE_KERNEL; 303 break; 304 case I915_MAP_WC: 305 pgprot = pgprot_writecombine(PAGE_KERNEL_IO); 306 break; 307 } 308 309 if (n_pages > ARRAY_SIZE(stack)) { 310 /* Too big for stack -- allocate temporary array instead */ 311 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); 312 if (!pages) 313 return ERR_PTR(-ENOMEM); 314 } 315 316 i = 0; 317 for_each_sgt_page(page, iter, obj->mm.pages) 318 pages[i++] = page; 319 vaddr = vmap(pages, n_pages, 0, pgprot); 320 if (pages != stack) 321 kvfree(pages); 322 323 return vaddr ?: ERR_PTR(-ENOMEM); 324 } 325 326 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, 327 enum i915_map_type type) 328 { 329 resource_size_t iomap = obj->mm.region->iomap.base - 330 obj->mm.region->region.start; 331 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT; 332 unsigned long stack[32], *pfns = stack, i; 333 struct sgt_iter iter; 334 dma_addr_t addr; 335 void *vaddr; 336 337 GEM_BUG_ON(type != I915_MAP_WC); 338 339 if (n_pfn > ARRAY_SIZE(stack)) { 340 /* Too big for stack -- allocate temporary array instead */ 341 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL); 342 if (!pfns) 343 return ERR_PTR(-ENOMEM); 344 } 345 346 i = 0; 347 for_each_sgt_daddr(addr, iter, obj->mm.pages) 348 pfns[i++] = (iomap + addr) >> PAGE_SHIFT; 349 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO)); 350 if (pfns != stack) 351 kvfree(pfns); 352 353 return vaddr ?: ERR_PTR(-ENOMEM); 354 } 355 356 /* get, pin, and map the pages of the object into kernel space */ 357 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 358 enum i915_map_type type) 359 { 360 enum i915_map_type has_type; 361 bool pinned; 362 void *ptr; 363 int err; 364 365 if (!i915_gem_object_has_struct_page(obj) && 366 !i915_gem_object_has_iomem(obj)) 367 return ERR_PTR(-ENXIO); 368 369 assert_object_held(obj); 370 371 pinned = !(type & I915_MAP_OVERRIDE); 372 type &= ~I915_MAP_OVERRIDE; 373 374 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { 375 if (unlikely(!i915_gem_object_has_pages(obj))) { 376 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 377 378 err = ____i915_gem_object_get_pages(obj); 379 if (err) 380 return ERR_PTR(err); 381 382 smp_mb__before_atomic(); 383 } 384 atomic_inc(&obj->mm.pages_pin_count); 385 pinned = false; 386 } 387 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 388 389 /* 390 * For discrete our CPU mappings needs to be consistent in order to 391 * function correctly on !x86. When mapping things through TTM, we use 392 * the same rules to determine the caching type. 393 * 394 * The caching rules, starting from DG1: 395 * 396 * - If the object can be placed in device local-memory, then the 397 * pages should be allocated and mapped as write-combined only. 398 * 399 * - Everything else is always allocated and mapped as write-back, 400 * with the guarantee that everything is also coherent with the 401 * GPU. 402 * 403 * Internal users of lmem are already expected to get this right, so no 404 * fudging needed there. 405 */ 406 if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) { 407 if (type != I915_MAP_WC && !obj->mm.n_placements) { 408 ptr = ERR_PTR(-ENODEV); 409 goto err_unpin; 410 } 411 412 type = I915_MAP_WC; 413 } else if (IS_DGFX(to_i915(obj->base.dev))) { 414 type = I915_MAP_WB; 415 } 416 417 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 418 if (ptr && has_type != type) { 419 if (pinned) { 420 ptr = ERR_PTR(-EBUSY); 421 goto err_unpin; 422 } 423 424 unmap_object(obj, ptr); 425 426 ptr = obj->mm.mapping = NULL; 427 } 428 429 if (!ptr) { 430 err = i915_gem_object_wait_moving_fence(obj, true); 431 if (err) { 432 ptr = ERR_PTR(err); 433 goto err_unpin; 434 } 435 436 if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled())) 437 ptr = ERR_PTR(-ENODEV); 438 else if (i915_gem_object_has_struct_page(obj)) 439 ptr = i915_gem_object_map_page(obj, type); 440 else 441 ptr = i915_gem_object_map_pfn(obj, type); 442 if (IS_ERR(ptr)) 443 goto err_unpin; 444 445 obj->mm.mapping = page_pack_bits(ptr, type); 446 } 447 448 return ptr; 449 450 err_unpin: 451 atomic_dec(&obj->mm.pages_pin_count); 452 return ptr; 453 } 454 455 void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 456 enum i915_map_type type) 457 { 458 void *ret; 459 460 i915_gem_object_lock(obj, NULL); 461 ret = i915_gem_object_pin_map(obj, type); 462 i915_gem_object_unlock(obj); 463 464 return ret; 465 } 466 467 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 468 unsigned long offset, 469 unsigned long size) 470 { 471 enum i915_map_type has_type; 472 void *ptr; 473 474 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 475 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), 476 offset, size, obj->base.size)); 477 478 wmb(); /* let all previous writes be visible to coherent partners */ 479 obj->mm.dirty = true; 480 481 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) 482 return; 483 484 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 485 if (has_type == I915_MAP_WC) 486 return; 487 488 drm_clflush_virt_range(ptr + offset, size); 489 if (size == obj->base.size) { 490 obj->write_domain &= ~I915_GEM_DOMAIN_CPU; 491 obj->cache_dirty = false; 492 } 493 } 494 495 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) 496 { 497 GEM_BUG_ON(!obj->mm.mapping); 498 499 /* 500 * We allow removing the mapping from underneath pinned pages! 501 * 502 * Furthermore, since this is an unsafe operation reserved only 503 * for construction time manipulation, we ignore locking prudence. 504 */ 505 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))); 506 507 i915_gem_object_unpin_map(obj); 508 } 509 510 struct scatterlist * 511 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 512 struct i915_gem_object_page_iter *iter, 513 unsigned int n, 514 unsigned int *offset, 515 bool dma) 516 { 517 struct scatterlist *sg; 518 unsigned int idx, count; 519 520 might_sleep(); 521 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); 522 if (!i915_gem_object_has_pinned_pages(obj)) 523 assert_object_held(obj); 524 525 /* As we iterate forward through the sg, we record each entry in a 526 * radixtree for quick repeated (backwards) lookups. If we have seen 527 * this index previously, we will have an entry for it. 528 * 529 * Initial lookup is O(N), but this is amortized to O(1) for 530 * sequential page access (where each new request is consecutive 531 * to the previous one). Repeated lookups are O(lg(obj->base.size)), 532 * i.e. O(1) with a large constant! 533 */ 534 if (n < READ_ONCE(iter->sg_idx)) 535 goto lookup; 536 537 mutex_lock(&iter->lock); 538 539 /* We prefer to reuse the last sg so that repeated lookup of this 540 * (or the subsequent) sg are fast - comparing against the last 541 * sg is faster than going through the radixtree. 542 */ 543 544 sg = iter->sg_pos; 545 idx = iter->sg_idx; 546 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 547 548 while (idx + count <= n) { 549 void *entry; 550 unsigned long i; 551 int ret; 552 553 /* If we cannot allocate and insert this entry, or the 554 * individual pages from this range, cancel updating the 555 * sg_idx so that on this lookup we are forced to linearly 556 * scan onwards, but on future lookups we will try the 557 * insertion again (in which case we need to be careful of 558 * the error return reporting that we have already inserted 559 * this index). 560 */ 561 ret = radix_tree_insert(&iter->radix, idx, sg); 562 if (ret && ret != -EEXIST) 563 goto scan; 564 565 entry = xa_mk_value(idx); 566 for (i = 1; i < count; i++) { 567 ret = radix_tree_insert(&iter->radix, idx + i, entry); 568 if (ret && ret != -EEXIST) 569 goto scan; 570 } 571 572 idx += count; 573 sg = ____sg_next(sg); 574 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 575 } 576 577 scan: 578 iter->sg_pos = sg; 579 iter->sg_idx = idx; 580 581 mutex_unlock(&iter->lock); 582 583 if (unlikely(n < idx)) /* insertion completed by another thread */ 584 goto lookup; 585 586 /* In case we failed to insert the entry into the radixtree, we need 587 * to look beyond the current sg. 588 */ 589 while (idx + count <= n) { 590 idx += count; 591 sg = ____sg_next(sg); 592 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 593 } 594 595 *offset = n - idx; 596 return sg; 597 598 lookup: 599 rcu_read_lock(); 600 601 sg = radix_tree_lookup(&iter->radix, n); 602 GEM_BUG_ON(!sg); 603 604 /* If this index is in the middle of multi-page sg entry, 605 * the radix tree will contain a value entry that points 606 * to the start of that range. We will return the pointer to 607 * the base page and the offset of this page within the 608 * sg entry's range. 609 */ 610 *offset = 0; 611 if (unlikely(xa_is_value(sg))) { 612 unsigned long base = xa_to_value(sg); 613 614 sg = radix_tree_lookup(&iter->radix, base); 615 GEM_BUG_ON(!sg); 616 617 *offset = n - base; 618 } 619 620 rcu_read_unlock(); 621 622 return sg; 623 } 624 625 struct page * 626 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) 627 { 628 struct scatterlist *sg; 629 unsigned int offset; 630 631 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 632 633 sg = i915_gem_object_get_sg(obj, n, &offset); 634 return nth_page(sg_page(sg), offset); 635 } 636 637 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 638 struct page * 639 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 640 unsigned int n) 641 { 642 struct page *page; 643 644 page = i915_gem_object_get_page(obj, n); 645 if (!obj->mm.dirty) 646 set_page_dirty(page); 647 648 return page; 649 } 650 651 dma_addr_t 652 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, 653 unsigned long n, 654 unsigned int *len) 655 { 656 struct scatterlist *sg; 657 unsigned int offset; 658 659 sg = i915_gem_object_get_sg_dma(obj, n, &offset); 660 661 if (len) 662 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); 663 664 return sg_dma_address(sg) + (offset << PAGE_SHIFT); 665 } 666 667 dma_addr_t 668 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 669 unsigned long n) 670 { 671 return i915_gem_object_get_dma_address_len(obj, n, NULL); 672 } 673