1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm_cache.h> 12 13 #include "gem/i915_gem_region.h" 14 #include "i915_drv.h" 15 #include "i915_gem_object.h" 16 #include "i915_gem_tiling.h" 17 #include "i915_gemfs.h" 18 #include "i915_scatterlist.h" 19 #include "i915_trace.h" 20 21 /* 22 * Move pages to appropriate lru and release the pagevec, decrementing the 23 * ref count of those pages. 24 */ 25 static void check_release_pagevec(struct pagevec *pvec) 26 { 27 check_move_unevictable_pages(pvec); 28 __pagevec_release(pvec); 29 cond_resched(); 30 } 31 32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 33 bool dirty, bool backup) 34 { 35 struct sgt_iter sgt_iter; 36 struct pagevec pvec; 37 struct page *page; 38 39 mapping_clear_unevictable(mapping); 40 41 pagevec_init(&pvec); 42 for_each_sgt_page(page, sgt_iter, st) { 43 if (dirty) 44 set_page_dirty(page); 45 46 if (backup) 47 mark_page_accessed(page); 48 49 if (!pagevec_add(&pvec, page)) 50 check_release_pagevec(&pvec); 51 } 52 if (pagevec_count(&pvec)) 53 check_release_pagevec(&pvec); 54 55 sg_free_table(st); 56 } 57 58 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 59 size_t size, struct intel_memory_region *mr, 60 struct address_space *mapping, 61 unsigned int max_segment) 62 { 63 unsigned int page_count; /* restricted by sg_alloc_table */ 64 unsigned long i; 65 struct scatterlist *sg; 66 struct page *page; 67 unsigned long last_pfn = 0; /* suppress gcc warning */ 68 gfp_t noreclaim; 69 int ret; 70 71 if (overflows_type(size / PAGE_SIZE, page_count)) 72 return -E2BIG; 73 74 page_count = size / PAGE_SIZE; 75 /* 76 * If there's no chance of allocating enough pages for the whole 77 * object, bail early. 78 */ 79 if (size > resource_size(&mr->region)) 80 return -ENOMEM; 81 82 if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN)) 83 return -ENOMEM; 84 85 /* 86 * Get the list of pages out of our struct file. They'll be pinned 87 * at this point until we release them. 88 * 89 * Fail silently without starting the shrinker 90 */ 91 mapping_set_unevictable(mapping); 92 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 93 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 94 95 sg = st->sgl; 96 st->nents = 0; 97 for (i = 0; i < page_count; i++) { 98 const unsigned int shrink[] = { 99 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 100 0, 101 }, *s = shrink; 102 gfp_t gfp = noreclaim; 103 104 do { 105 cond_resched(); 106 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 107 if (!IS_ERR(page)) 108 break; 109 110 if (!*s) { 111 ret = PTR_ERR(page); 112 goto err_sg; 113 } 114 115 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++); 116 117 /* 118 * We've tried hard to allocate the memory by reaping 119 * our own buffer, now let the real VM do its job and 120 * go down in flames if truly OOM. 121 * 122 * However, since graphics tend to be disposable, 123 * defer the oom here by reporting the ENOMEM back 124 * to userspace. 125 */ 126 if (!*s) { 127 /* reclaim and warn, but no oom */ 128 gfp = mapping_gfp_mask(mapping); 129 130 /* 131 * Our bo are always dirty and so we require 132 * kswapd to reclaim our pages (direct reclaim 133 * does not effectively begin pageout of our 134 * buffers on its own). However, direct reclaim 135 * only waits for kswapd when under allocation 136 * congestion. So as a result __GFP_RECLAIM is 137 * unreliable and fails to actually reclaim our 138 * dirty pages -- unless you try over and over 139 * again with !__GFP_NORETRY. However, we still 140 * want to fail this allocation rather than 141 * trigger the out-of-memory killer and for 142 * this we want __GFP_RETRY_MAYFAIL. 143 */ 144 gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; 145 } 146 } while (1); 147 148 if (!i || 149 sg->length >= max_segment || 150 page_to_pfn(page) != last_pfn + 1) { 151 if (i) 152 sg = sg_next(sg); 153 154 st->nents++; 155 sg_set_page(sg, page, PAGE_SIZE, 0); 156 } else { 157 sg->length += PAGE_SIZE; 158 } 159 last_pfn = page_to_pfn(page); 160 161 /* Check that the i965g/gm workaround works. */ 162 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL); 163 } 164 if (sg) /* loop terminated early; short sg table */ 165 sg_mark_end(sg); 166 167 /* Trim unused sg entries to avoid wasting memory. */ 168 i915_sg_trim(st); 169 170 return 0; 171 err_sg: 172 sg_mark_end(sg); 173 if (sg != st->sgl) { 174 shmem_sg_free_table(st, mapping, false, false); 175 } else { 176 mapping_clear_unevictable(mapping); 177 sg_free_table(st); 178 } 179 180 /* 181 * shmemfs first checks if there is enough memory to allocate the page 182 * and reports ENOSPC should there be insufficient, along with the usual 183 * ENOMEM for a genuine allocation failure. 184 * 185 * We use ENOSPC in our driver to mean that we have run out of aperture 186 * space and so want to translate the error from shmemfs back to our 187 * usual understanding of ENOMEM. 188 */ 189 if (ret == -ENOSPC) 190 ret = -ENOMEM; 191 192 return ret; 193 } 194 195 static int shmem_get_pages(struct drm_i915_gem_object *obj) 196 { 197 struct drm_i915_private *i915 = to_i915(obj->base.dev); 198 struct intel_memory_region *mem = obj->mm.region; 199 struct address_space *mapping = obj->base.filp->f_mapping; 200 unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); 201 struct sg_table *st; 202 struct sgt_iter sgt_iter; 203 struct page *page; 204 int ret; 205 206 /* 207 * Assert that the object is not currently in any GPU domain. As it 208 * wasn't in the GTT, there shouldn't be any way it could have been in 209 * a GPU cache 210 */ 211 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 212 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 213 214 rebuild_st: 215 st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN); 216 if (!st) 217 return -ENOMEM; 218 219 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping, 220 max_segment); 221 if (ret) 222 goto err_st; 223 224 ret = i915_gem_gtt_prepare_pages(obj, st); 225 if (ret) { 226 /* 227 * DMA remapping failed? One possible cause is that 228 * it could not reserve enough large entries, asking 229 * for PAGE_SIZE chunks instead may be helpful. 230 */ 231 if (max_segment > PAGE_SIZE) { 232 for_each_sgt_page(page, sgt_iter, st) 233 put_page(page); 234 sg_free_table(st); 235 kfree(st); 236 237 max_segment = PAGE_SIZE; 238 goto rebuild_st; 239 } else { 240 dev_warn(i915->drm.dev, 241 "Failed to DMA remap %lu pages\n", 242 obj->base.size >> PAGE_SHIFT); 243 goto err_pages; 244 } 245 } 246 247 if (i915_gem_object_needs_bit17_swizzle(obj)) 248 i915_gem_object_do_bit_17_swizzle(obj, st); 249 250 if (i915_gem_object_can_bypass_llc(obj)) 251 obj->cache_dirty = true; 252 253 __i915_gem_object_set_pages(obj, st); 254 255 return 0; 256 257 err_pages: 258 shmem_sg_free_table(st, mapping, false, false); 259 /* 260 * shmemfs first checks if there is enough memory to allocate the page 261 * and reports ENOSPC should there be insufficient, along with the usual 262 * ENOMEM for a genuine allocation failure. 263 * 264 * We use ENOSPC in our driver to mean that we have run out of aperture 265 * space and so want to translate the error from shmemfs back to our 266 * usual understanding of ENOMEM. 267 */ 268 err_st: 269 if (ret == -ENOSPC) 270 ret = -ENOMEM; 271 272 kfree(st); 273 274 return ret; 275 } 276 277 static int 278 shmem_truncate(struct drm_i915_gem_object *obj) 279 { 280 /* 281 * Our goal here is to return as much of the memory as 282 * is possible back to the system as we are called from OOM. 283 * To do this we must instruct the shmfs to drop all of its 284 * backing pages, *now*. 285 */ 286 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 287 obj->mm.madv = __I915_MADV_PURGED; 288 obj->mm.pages = ERR_PTR(-EFAULT); 289 290 return 0; 291 } 292 293 void __shmem_writeback(size_t size, struct address_space *mapping) 294 { 295 struct writeback_control wbc = { 296 .sync_mode = WB_SYNC_NONE, 297 .nr_to_write = SWAP_CLUSTER_MAX, 298 .range_start = 0, 299 .range_end = LLONG_MAX, 300 .for_reclaim = 1, 301 }; 302 unsigned long i; 303 304 /* 305 * Leave mmapings intact (GTT will have been revoked on unbinding, 306 * leaving only CPU mmapings around) and add those pages to the LRU 307 * instead of invoking writeback so they are aged and paged out 308 * as normal. 309 */ 310 311 /* Begin writeback on each dirty page */ 312 for (i = 0; i < size >> PAGE_SHIFT; i++) { 313 struct page *page; 314 315 page = find_lock_page(mapping, i); 316 if (!page) 317 continue; 318 319 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 320 int ret; 321 322 SetPageReclaim(page); 323 ret = mapping->a_ops->writepage(page, &wbc); 324 if (!PageWriteback(page)) 325 ClearPageReclaim(page); 326 if (!ret) 327 goto put; 328 } 329 unlock_page(page); 330 put: 331 put_page(page); 332 } 333 } 334 335 static void 336 shmem_writeback(struct drm_i915_gem_object *obj) 337 { 338 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping); 339 } 340 341 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags) 342 { 343 switch (obj->mm.madv) { 344 case I915_MADV_DONTNEED: 345 return i915_gem_object_truncate(obj); 346 case __I915_MADV_PURGED: 347 return 0; 348 } 349 350 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) 351 shmem_writeback(obj); 352 353 return 0; 354 } 355 356 void 357 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 358 struct sg_table *pages, 359 bool needs_clflush) 360 { 361 struct drm_i915_private *i915 = to_i915(obj->base.dev); 362 363 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 364 365 if (obj->mm.madv == I915_MADV_DONTNEED) 366 obj->mm.dirty = false; 367 368 if (needs_clflush && 369 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 370 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 371 drm_clflush_sg(pages); 372 373 __start_cpu_write(obj); 374 /* 375 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever 376 * swapped-in. Our async flush path is not trust worthy enough yet(and 377 * happens in the wrong order), and with some tricks it's conceivable 378 * for userspace to change the cache-level to I915_CACHE_NONE after the 379 * pages are swapped-in, and since execbuf binds the object before doing 380 * the async flush, we have a race window. 381 */ 382 if (!HAS_LLC(i915) && !IS_DGFX(i915)) 383 obj->cache_dirty = true; 384 } 385 386 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) 387 { 388 __i915_gem_object_release_shmem(obj, pages, true); 389 390 i915_gem_gtt_finish_pages(obj, pages); 391 392 if (i915_gem_object_needs_bit17_swizzle(obj)) 393 i915_gem_object_save_bit_17_swizzle(obj, pages); 394 395 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping, 396 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED); 397 kfree(pages); 398 obj->mm.dirty = false; 399 } 400 401 static void 402 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 403 { 404 if (likely(i915_gem_object_has_struct_page(obj))) 405 i915_gem_object_put_pages_shmem(obj, pages); 406 else 407 i915_gem_object_put_pages_phys(obj, pages); 408 } 409 410 static int 411 shmem_pwrite(struct drm_i915_gem_object *obj, 412 const struct drm_i915_gem_pwrite *arg) 413 { 414 struct address_space *mapping = obj->base.filp->f_mapping; 415 const struct address_space_operations *aops = mapping->a_ops; 416 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 417 u64 remain, offset; 418 unsigned int pg; 419 420 /* Caller already validated user args */ 421 GEM_BUG_ON(!access_ok(user_data, arg->size)); 422 423 if (!i915_gem_object_has_struct_page(obj)) 424 return i915_gem_object_pwrite_phys(obj, arg); 425 426 /* 427 * Before we instantiate/pin the backing store for our use, we 428 * can prepopulate the shmemfs filp efficiently using a write into 429 * the pagecache. We avoid the penalty of instantiating all the 430 * pages, important if the user is just writing to a few and never 431 * uses the object on the GPU, and using a direct write into shmemfs 432 * allows it to avoid the cost of retrieving a page (either swapin 433 * or clearing-before-use) before it is overwritten. 434 */ 435 if (i915_gem_object_has_pages(obj)) 436 return -ENODEV; 437 438 if (obj->mm.madv != I915_MADV_WILLNEED) 439 return -EFAULT; 440 441 /* 442 * Before the pages are instantiated the object is treated as being 443 * in the CPU domain. The pages will be clflushed as required before 444 * use, and we can freely write into the pages directly. If userspace 445 * races pwrite with any other operation; corruption will ensue - 446 * that is userspace's prerogative! 447 */ 448 449 remain = arg->size; 450 offset = arg->offset; 451 pg = offset_in_page(offset); 452 453 do { 454 unsigned int len, unwritten; 455 struct page *page; 456 void *data, *vaddr; 457 int err; 458 char c; 459 460 len = PAGE_SIZE - pg; 461 if (len > remain) 462 len = remain; 463 464 /* Prefault the user page to reduce potential recursion */ 465 err = __get_user(c, user_data); 466 if (err) 467 return err; 468 469 err = __get_user(c, user_data + len - 1); 470 if (err) 471 return err; 472 473 err = aops->write_begin(obj->base.filp, mapping, offset, len, 474 &page, &data); 475 if (err < 0) 476 return err; 477 478 vaddr = kmap_atomic(page); 479 unwritten = __copy_from_user_inatomic(vaddr + pg, 480 user_data, 481 len); 482 kunmap_atomic(vaddr); 483 484 err = aops->write_end(obj->base.filp, mapping, offset, len, 485 len - unwritten, page, data); 486 if (err < 0) 487 return err; 488 489 /* We don't handle -EFAULT, leave it to the caller to check */ 490 if (unwritten) 491 return -ENODEV; 492 493 remain -= len; 494 user_data += len; 495 offset += len; 496 pg = 0; 497 } while (remain); 498 499 return 0; 500 } 501 502 static int 503 shmem_pread(struct drm_i915_gem_object *obj, 504 const struct drm_i915_gem_pread *arg) 505 { 506 if (!i915_gem_object_has_struct_page(obj)) 507 return i915_gem_object_pread_phys(obj, arg); 508 509 return -ENODEV; 510 } 511 512 static void shmem_release(struct drm_i915_gem_object *obj) 513 { 514 if (i915_gem_object_has_struct_page(obj)) 515 i915_gem_object_release_memory_region(obj); 516 517 fput(obj->base.filp); 518 } 519 520 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 521 .name = "i915_gem_object_shmem", 522 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 523 524 .get_pages = shmem_get_pages, 525 .put_pages = shmem_put_pages, 526 .truncate = shmem_truncate, 527 .shrink = shmem_shrink, 528 529 .pwrite = shmem_pwrite, 530 .pread = shmem_pread, 531 532 .release = shmem_release, 533 }; 534 535 static int __create_shmem(struct drm_i915_private *i915, 536 struct drm_gem_object *obj, 537 resource_size_t size) 538 { 539 unsigned long flags = VM_NORESERVE; 540 struct file *filp; 541 542 drm_gem_private_object_init(&i915->drm, obj, size); 543 544 if (i915->mm.gemfs) 545 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 546 flags); 547 else 548 filp = shmem_file_setup("i915", size, flags); 549 if (IS_ERR(filp)) 550 return PTR_ERR(filp); 551 552 obj->filp = filp; 553 return 0; 554 } 555 556 static int shmem_object_init(struct intel_memory_region *mem, 557 struct drm_i915_gem_object *obj, 558 resource_size_t offset, 559 resource_size_t size, 560 resource_size_t page_size, 561 unsigned int flags) 562 { 563 static struct lock_class_key lock_class; 564 struct drm_i915_private *i915 = mem->i915; 565 struct address_space *mapping; 566 unsigned int cache_level; 567 gfp_t mask; 568 int ret; 569 570 ret = __create_shmem(i915, &obj->base, size); 571 if (ret) 572 return ret; 573 574 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 575 if (IS_I965GM(i915) || IS_I965G(i915)) { 576 /* 965gm cannot relocate objects above 4GiB. */ 577 mask &= ~__GFP_HIGHMEM; 578 mask |= __GFP_DMA32; 579 } 580 581 mapping = obj->base.filp->f_mapping; 582 mapping_set_gfp_mask(mapping, mask); 583 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 584 585 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0); 586 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; 587 obj->write_domain = I915_GEM_DOMAIN_CPU; 588 obj->read_domains = I915_GEM_DOMAIN_CPU; 589 590 if (HAS_LLC(i915)) 591 /* On some devices, we can have the GPU use the LLC (the CPU 592 * cache) for about a 10% performance improvement 593 * compared to uncached. Graphics requests other than 594 * display scanout are coherent with the CPU in 595 * accessing this cache. This means in this mode we 596 * don't need to clflush on the CPU side, and on the 597 * GPU side we only need to flush internal caches to 598 * get data visible to the CPU. 599 * 600 * However, we maintain the display planes as UC, and so 601 * need to rebind when first used as such. 602 */ 603 cache_level = I915_CACHE_LLC; 604 else 605 cache_level = I915_CACHE_NONE; 606 607 i915_gem_object_set_cache_coherency(obj, cache_level); 608 609 i915_gem_object_init_memory_region(obj, mem); 610 611 return 0; 612 } 613 614 struct drm_i915_gem_object * 615 i915_gem_object_create_shmem(struct drm_i915_private *i915, 616 resource_size_t size) 617 { 618 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 619 size, 0, 0); 620 } 621 622 /* Allocate a new GEM object and fill it with the supplied data */ 623 struct drm_i915_gem_object * 624 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 625 const void *data, resource_size_t size) 626 { 627 struct drm_i915_gem_object *obj; 628 struct file *file; 629 const struct address_space_operations *aops; 630 resource_size_t offset; 631 int err; 632 633 GEM_WARN_ON(IS_DGFX(dev_priv)); 634 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 635 if (IS_ERR(obj)) 636 return obj; 637 638 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 639 640 file = obj->base.filp; 641 aops = file->f_mapping->a_ops; 642 offset = 0; 643 do { 644 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 645 struct page *page; 646 void *pgdata, *vaddr; 647 648 err = aops->write_begin(file, file->f_mapping, offset, len, 649 &page, &pgdata); 650 if (err < 0) 651 goto fail; 652 653 vaddr = kmap(page); 654 memcpy(vaddr, data, len); 655 kunmap(page); 656 657 err = aops->write_end(file, file->f_mapping, offset, len, len, 658 page, pgdata); 659 if (err < 0) 660 goto fail; 661 662 size -= len; 663 data += len; 664 offset += len; 665 } while (size); 666 667 return obj; 668 669 fail: 670 i915_gem_object_put(obj); 671 return ERR_PTR(err); 672 } 673 674 static int init_shmem(struct intel_memory_region *mem) 675 { 676 i915_gemfs_init(mem->i915); 677 intel_memory_region_set_name(mem, "system"); 678 679 return 0; /* We have fallback to the kernel mnt if gemfs init failed. */ 680 } 681 682 static int release_shmem(struct intel_memory_region *mem) 683 { 684 i915_gemfs_fini(mem->i915); 685 return 0; 686 } 687 688 static const struct intel_memory_region_ops shmem_region_ops = { 689 .init = init_shmem, 690 .release = release_shmem, 691 .init_object = shmem_object_init, 692 }; 693 694 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, 695 u16 type, u16 instance) 696 { 697 return intel_memory_region_create(i915, 0, 698 totalram_pages() << PAGE_SHIFT, 699 PAGE_SIZE, 0, 0, 700 type, instance, 701 &shmem_region_ops); 702 } 703 704 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj) 705 { 706 return obj->ops == &i915_gem_shmem_ops; 707 } 708