1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "i915_gem_object.h" 9 #include "i915_scatterlist.h" 10 #include "i915_gem_lmem.h" 11 #include "i915_gem_mman.h" 12 13 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 14 struct sg_table *pages, 15 unsigned int sg_page_sizes) 16 { 17 struct drm_i915_private *i915 = to_i915(obj->base.dev); 18 unsigned long supported = INTEL_INFO(i915)->page_sizes; 19 bool shrinkable; 20 int i; 21 22 assert_object_held_shared(obj); 23 24 if (i915_gem_object_is_volatile(obj)) 25 obj->mm.madv = I915_MADV_DONTNEED; 26 27 /* Make the pages coherent with the GPU (flushing any swapin). */ 28 if (obj->cache_dirty) { 29 obj->write_domain = 0; 30 if (i915_gem_object_has_struct_page(obj)) 31 drm_clflush_sg(pages); 32 obj->cache_dirty = false; 33 } 34 35 obj->mm.get_page.sg_pos = pages->sgl; 36 obj->mm.get_page.sg_idx = 0; 37 obj->mm.get_dma_page.sg_pos = pages->sgl; 38 obj->mm.get_dma_page.sg_idx = 0; 39 40 obj->mm.pages = pages; 41 42 GEM_BUG_ON(!sg_page_sizes); 43 obj->mm.page_sizes.phys = sg_page_sizes; 44 45 /* 46 * Calculate the supported page-sizes which fit into the given 47 * sg_page_sizes. This will give us the page-sizes which we may be able 48 * to use opportunistically when later inserting into the GTT. For 49 * example if phys=2G, then in theory we should be able to use 1G, 2M, 50 * 64K or 4K pages, although in practice this will depend on a number of 51 * other factors. 52 */ 53 obj->mm.page_sizes.sg = 0; 54 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 55 if (obj->mm.page_sizes.phys & ~0u << i) 56 obj->mm.page_sizes.sg |= BIT(i); 57 } 58 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); 59 60 shrinkable = i915_gem_object_is_shrinkable(obj); 61 62 if (i915_gem_object_is_tiled(obj) && 63 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 64 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); 65 i915_gem_object_set_tiling_quirk(obj); 66 GEM_BUG_ON(!list_empty(&obj->mm.link)); 67 atomic_inc(&obj->mm.shrink_pin); 68 shrinkable = false; 69 } 70 71 if (shrinkable) { 72 struct list_head *list; 73 unsigned long flags; 74 75 assert_object_held(obj); 76 spin_lock_irqsave(&i915->mm.obj_lock, flags); 77 78 i915->mm.shrink_count++; 79 i915->mm.shrink_memory += obj->base.size; 80 81 if (obj->mm.madv != I915_MADV_WILLNEED) 82 list = &i915->mm.purge_list; 83 else 84 list = &i915->mm.shrink_list; 85 list_add_tail(&obj->mm.link, list); 86 87 atomic_set(&obj->mm.shrink_pin, 0); 88 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 89 } 90 } 91 92 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 93 { 94 struct drm_i915_private *i915 = to_i915(obj->base.dev); 95 int err; 96 97 assert_object_held_shared(obj); 98 99 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 100 drm_dbg(&i915->drm, 101 "Attempting to obtain a purgeable object\n"); 102 return -EFAULT; 103 } 104 105 err = obj->ops->get_pages(obj); 106 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); 107 108 return err; 109 } 110 111 /* Ensure that the associated pages are gathered from the backing storage 112 * and pinned into our object. i915_gem_object_pin_pages() may be called 113 * multiple times before they are released by a single call to 114 * i915_gem_object_unpin_pages() - once the pages are no longer referenced 115 * either as a result of memory pressure (reaping pages under the shrinker) 116 * or as the object is itself released. 117 */ 118 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 119 { 120 int err; 121 122 assert_object_held(obj); 123 124 assert_object_held_shared(obj); 125 126 if (unlikely(!i915_gem_object_has_pages(obj))) { 127 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 128 129 err = ____i915_gem_object_get_pages(obj); 130 if (err) 131 return err; 132 133 smp_mb__before_atomic(); 134 } 135 atomic_inc(&obj->mm.pages_pin_count); 136 137 return 0; 138 } 139 140 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj) 141 { 142 struct i915_gem_ww_ctx ww; 143 int err; 144 145 i915_gem_ww_ctx_init(&ww, true); 146 retry: 147 err = i915_gem_object_lock(obj, &ww); 148 if (!err) 149 err = i915_gem_object_pin_pages(obj); 150 151 if (err == -EDEADLK) { 152 err = i915_gem_ww_ctx_backoff(&ww); 153 if (!err) 154 goto retry; 155 } 156 i915_gem_ww_ctx_fini(&ww); 157 return err; 158 } 159 160 /* Immediately discard the backing storage */ 161 void i915_gem_object_truncate(struct drm_i915_gem_object *obj) 162 { 163 drm_gem_free_mmap_offset(&obj->base); 164 if (obj->ops->truncate) 165 obj->ops->truncate(obj); 166 } 167 168 /* Try to discard unwanted pages */ 169 void i915_gem_object_writeback(struct drm_i915_gem_object *obj) 170 { 171 assert_object_held_shared(obj); 172 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 173 174 if (obj->ops->writeback) 175 obj->ops->writeback(obj); 176 } 177 178 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) 179 { 180 struct radix_tree_iter iter; 181 void __rcu **slot; 182 183 rcu_read_lock(); 184 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) 185 radix_tree_delete(&obj->mm.get_page.radix, iter.index); 186 radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0) 187 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index); 188 rcu_read_unlock(); 189 } 190 191 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr) 192 { 193 if (is_vmalloc_addr(ptr)) 194 vunmap(ptr); 195 } 196 197 struct sg_table * 198 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) 199 { 200 struct sg_table *pages; 201 202 assert_object_held_shared(obj); 203 204 pages = fetch_and_zero(&obj->mm.pages); 205 if (IS_ERR_OR_NULL(pages)) 206 return pages; 207 208 if (i915_gem_object_is_volatile(obj)) 209 obj->mm.madv = I915_MADV_WILLNEED; 210 211 i915_gem_object_make_unshrinkable(obj); 212 213 if (obj->mm.mapping) { 214 unmap_object(obj, page_mask_bits(obj->mm.mapping)); 215 obj->mm.mapping = NULL; 216 } 217 218 __i915_gem_object_reset_page_iter(obj); 219 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; 220 221 return pages; 222 } 223 224 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) 225 { 226 struct sg_table *pages; 227 228 if (i915_gem_object_has_pinned_pages(obj)) 229 return -EBUSY; 230 231 /* May be called by shrinker from within get_pages() (on another bo) */ 232 assert_object_held_shared(obj); 233 234 i915_gem_object_release_mmap_offset(obj); 235 236 /* 237 * ->put_pages might need to allocate memory for the bit17 swizzle 238 * array, hence protect them from being reaped by removing them from gtt 239 * lists early. 240 */ 241 pages = __i915_gem_object_unset_pages(obj); 242 243 /* 244 * XXX Temporary hijinx to avoid updating all backends to handle 245 * NULL pages. In the future, when we have more asynchronous 246 * get_pages backends we should be better able to handle the 247 * cancellation of the async task in a more uniform manner. 248 */ 249 if (!IS_ERR_OR_NULL(pages)) 250 obj->ops->put_pages(obj, pages); 251 252 return 0; 253 } 254 255 /* The 'mapping' part of i915_gem_object_pin_map() below */ 256 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj, 257 enum i915_map_type type) 258 { 259 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i; 260 struct page *stack[32], **pages = stack, *page; 261 struct sgt_iter iter; 262 pgprot_t pgprot; 263 void *vaddr; 264 265 switch (type) { 266 default: 267 MISSING_CASE(type); 268 fallthrough; /* to use PAGE_KERNEL anyway */ 269 case I915_MAP_WB: 270 /* 271 * On 32b, highmem using a finite set of indirect PTE (i.e. 272 * vmap) to provide virtual mappings of the high pages. 273 * As these are finite, map_new_virtual() must wait for some 274 * other kmap() to finish when it runs out. If we map a large 275 * number of objects, there is no method for it to tell us 276 * to release the mappings, and we deadlock. 277 * 278 * However, if we make an explicit vmap of the page, that 279 * uses a larger vmalloc arena, and also has the ability 280 * to tell us to release unwanted mappings. Most importantly, 281 * it will fail and propagate an error instead of waiting 282 * forever. 283 * 284 * So if the page is beyond the 32b boundary, make an explicit 285 * vmap. 286 */ 287 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))) 288 return page_address(sg_page(obj->mm.pages->sgl)); 289 pgprot = PAGE_KERNEL; 290 break; 291 case I915_MAP_WC: 292 pgprot = pgprot_writecombine(PAGE_KERNEL_IO); 293 break; 294 } 295 296 if (n_pages > ARRAY_SIZE(stack)) { 297 /* Too big for stack -- allocate temporary array instead */ 298 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); 299 if (!pages) 300 return ERR_PTR(-ENOMEM); 301 } 302 303 i = 0; 304 for_each_sgt_page(page, iter, obj->mm.pages) 305 pages[i++] = page; 306 vaddr = vmap(pages, n_pages, 0, pgprot); 307 if (pages != stack) 308 kvfree(pages); 309 310 return vaddr ?: ERR_PTR(-ENOMEM); 311 } 312 313 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, 314 enum i915_map_type type) 315 { 316 resource_size_t iomap = obj->mm.region->iomap.base - 317 obj->mm.region->region.start; 318 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT; 319 unsigned long stack[32], *pfns = stack, i; 320 struct sgt_iter iter; 321 dma_addr_t addr; 322 void *vaddr; 323 324 if (type != I915_MAP_WC) 325 return ERR_PTR(-ENODEV); 326 327 if (n_pfn > ARRAY_SIZE(stack)) { 328 /* Too big for stack -- allocate temporary array instead */ 329 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL); 330 if (!pfns) 331 return ERR_PTR(-ENOMEM); 332 } 333 334 i = 0; 335 for_each_sgt_daddr(addr, iter, obj->mm.pages) 336 pfns[i++] = (iomap + addr) >> PAGE_SHIFT; 337 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO)); 338 if (pfns != stack) 339 kvfree(pfns); 340 341 return vaddr ?: ERR_PTR(-ENOMEM); 342 } 343 344 /* get, pin, and map the pages of the object into kernel space */ 345 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 346 enum i915_map_type type) 347 { 348 enum i915_map_type has_type; 349 bool pinned; 350 void *ptr; 351 int err; 352 353 if (!i915_gem_object_has_struct_page(obj) && 354 !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) 355 return ERR_PTR(-ENXIO); 356 357 assert_object_held(obj); 358 359 pinned = !(type & I915_MAP_OVERRIDE); 360 type &= ~I915_MAP_OVERRIDE; 361 362 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { 363 if (unlikely(!i915_gem_object_has_pages(obj))) { 364 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 365 366 err = ____i915_gem_object_get_pages(obj); 367 if (err) 368 return ERR_PTR(err); 369 370 smp_mb__before_atomic(); 371 } 372 atomic_inc(&obj->mm.pages_pin_count); 373 pinned = false; 374 } 375 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 376 377 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 378 if (ptr && has_type != type) { 379 if (pinned) { 380 ptr = ERR_PTR(-EBUSY); 381 goto err_unpin; 382 } 383 384 unmap_object(obj, ptr); 385 386 ptr = obj->mm.mapping = NULL; 387 } 388 389 if (!ptr) { 390 if (GEM_WARN_ON(type == I915_MAP_WC && 391 !static_cpu_has(X86_FEATURE_PAT))) 392 ptr = ERR_PTR(-ENODEV); 393 else if (i915_gem_object_has_struct_page(obj)) 394 ptr = i915_gem_object_map_page(obj, type); 395 else 396 ptr = i915_gem_object_map_pfn(obj, type); 397 if (IS_ERR(ptr)) 398 goto err_unpin; 399 400 obj->mm.mapping = page_pack_bits(ptr, type); 401 } 402 403 return ptr; 404 405 err_unpin: 406 atomic_dec(&obj->mm.pages_pin_count); 407 return ptr; 408 } 409 410 void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 411 enum i915_map_type type) 412 { 413 void *ret; 414 415 i915_gem_object_lock(obj, NULL); 416 ret = i915_gem_object_pin_map(obj, type); 417 i915_gem_object_unlock(obj); 418 419 return ret; 420 } 421 422 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 423 unsigned long offset, 424 unsigned long size) 425 { 426 enum i915_map_type has_type; 427 void *ptr; 428 429 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 430 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), 431 offset, size, obj->base.size)); 432 433 wmb(); /* let all previous writes be visible to coherent partners */ 434 obj->mm.dirty = true; 435 436 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) 437 return; 438 439 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 440 if (has_type == I915_MAP_WC) 441 return; 442 443 drm_clflush_virt_range(ptr + offset, size); 444 if (size == obj->base.size) { 445 obj->write_domain &= ~I915_GEM_DOMAIN_CPU; 446 obj->cache_dirty = false; 447 } 448 } 449 450 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) 451 { 452 GEM_BUG_ON(!obj->mm.mapping); 453 454 /* 455 * We allow removing the mapping from underneath pinned pages! 456 * 457 * Furthermore, since this is an unsafe operation reserved only 458 * for construction time manipulation, we ignore locking prudence. 459 */ 460 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))); 461 462 i915_gem_object_unpin_map(obj); 463 } 464 465 struct scatterlist * 466 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 467 struct i915_gem_object_page_iter *iter, 468 unsigned int n, 469 unsigned int *offset, 470 bool allow_alloc) 471 { 472 const bool dma = iter == &obj->mm.get_dma_page; 473 struct scatterlist *sg; 474 unsigned int idx, count; 475 476 might_sleep(); 477 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); 478 if (!i915_gem_object_has_pinned_pages(obj)) 479 assert_object_held(obj); 480 481 /* As we iterate forward through the sg, we record each entry in a 482 * radixtree for quick repeated (backwards) lookups. If we have seen 483 * this index previously, we will have an entry for it. 484 * 485 * Initial lookup is O(N), but this is amortized to O(1) for 486 * sequential page access (where each new request is consecutive 487 * to the previous one). Repeated lookups are O(lg(obj->base.size)), 488 * i.e. O(1) with a large constant! 489 */ 490 if (n < READ_ONCE(iter->sg_idx)) 491 goto lookup; 492 493 if (!allow_alloc) 494 goto manual_lookup; 495 496 mutex_lock(&iter->lock); 497 498 /* We prefer to reuse the last sg so that repeated lookup of this 499 * (or the subsequent) sg are fast - comparing against the last 500 * sg is faster than going through the radixtree. 501 */ 502 503 sg = iter->sg_pos; 504 idx = iter->sg_idx; 505 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 506 507 while (idx + count <= n) { 508 void *entry; 509 unsigned long i; 510 int ret; 511 512 /* If we cannot allocate and insert this entry, or the 513 * individual pages from this range, cancel updating the 514 * sg_idx so that on this lookup we are forced to linearly 515 * scan onwards, but on future lookups we will try the 516 * insertion again (in which case we need to be careful of 517 * the error return reporting that we have already inserted 518 * this index). 519 */ 520 ret = radix_tree_insert(&iter->radix, idx, sg); 521 if (ret && ret != -EEXIST) 522 goto scan; 523 524 entry = xa_mk_value(idx); 525 for (i = 1; i < count; i++) { 526 ret = radix_tree_insert(&iter->radix, idx + i, entry); 527 if (ret && ret != -EEXIST) 528 goto scan; 529 } 530 531 idx += count; 532 sg = ____sg_next(sg); 533 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 534 } 535 536 scan: 537 iter->sg_pos = sg; 538 iter->sg_idx = idx; 539 540 mutex_unlock(&iter->lock); 541 542 if (unlikely(n < idx)) /* insertion completed by another thread */ 543 goto lookup; 544 545 goto manual_walk; 546 547 manual_lookup: 548 idx = 0; 549 sg = obj->mm.pages->sgl; 550 count = __sg_page_count(sg); 551 552 manual_walk: 553 /* 554 * In case we failed to insert the entry into the radixtree, we need 555 * to look beyond the current sg. 556 */ 557 while (idx + count <= n) { 558 idx += count; 559 sg = ____sg_next(sg); 560 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); 561 } 562 563 *offset = n - idx; 564 return sg; 565 566 lookup: 567 rcu_read_lock(); 568 569 sg = radix_tree_lookup(&iter->radix, n); 570 GEM_BUG_ON(!sg); 571 572 /* If this index is in the middle of multi-page sg entry, 573 * the radix tree will contain a value entry that points 574 * to the start of that range. We will return the pointer to 575 * the base page and the offset of this page within the 576 * sg entry's range. 577 */ 578 *offset = 0; 579 if (unlikely(xa_is_value(sg))) { 580 unsigned long base = xa_to_value(sg); 581 582 sg = radix_tree_lookup(&iter->radix, base); 583 GEM_BUG_ON(!sg); 584 585 *offset = n - base; 586 } 587 588 rcu_read_unlock(); 589 590 return sg; 591 } 592 593 struct page * 594 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) 595 { 596 struct scatterlist *sg; 597 unsigned int offset; 598 599 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 600 601 sg = i915_gem_object_get_sg(obj, n, &offset, true); 602 return nth_page(sg_page(sg), offset); 603 } 604 605 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 606 struct page * 607 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 608 unsigned int n) 609 { 610 struct page *page; 611 612 page = i915_gem_object_get_page(obj, n); 613 if (!obj->mm.dirty) 614 set_page_dirty(page); 615 616 return page; 617 } 618 619 dma_addr_t 620 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, 621 unsigned long n, 622 unsigned int *len) 623 { 624 struct scatterlist *sg; 625 unsigned int offset; 626 627 sg = i915_gem_object_get_sg_dma(obj, n, &offset, true); 628 629 if (len) 630 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); 631 632 return sg_dma_address(sg) + (offset << PAGE_SHIFT); 633 } 634 635 dma_addr_t 636 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 637 unsigned long n) 638 { 639 return i915_gem_object_get_dma_address_len(obj, n, NULL); 640 } 641