1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm_cache.h> 12 13 #include "gem/i915_gem_region.h" 14 #include "i915_drv.h" 15 #include "i915_gem_object.h" 16 #include "i915_gem_tiling.h" 17 #include "i915_gemfs.h" 18 #include "i915_scatterlist.h" 19 #include "i915_trace.h" 20 21 /* 22 * Move pages to appropriate lru and release the pagevec, decrementing the 23 * ref count of those pages. 24 */ 25 static void check_release_pagevec(struct pagevec *pvec) 26 { 27 check_move_unevictable_pages(pvec); 28 __pagevec_release(pvec); 29 cond_resched(); 30 } 31 32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 33 bool dirty, bool backup) 34 { 35 struct sgt_iter sgt_iter; 36 struct pagevec pvec; 37 struct page *page; 38 39 mapping_clear_unevictable(mapping); 40 41 pagevec_init(&pvec); 42 for_each_sgt_page(page, sgt_iter, st) { 43 if (dirty) 44 set_page_dirty(page); 45 46 if (backup) 47 mark_page_accessed(page); 48 49 if (!pagevec_add(&pvec, page)) 50 check_release_pagevec(&pvec); 51 } 52 if (pagevec_count(&pvec)) 53 check_release_pagevec(&pvec); 54 55 sg_free_table(st); 56 } 57 58 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 59 size_t size, struct intel_memory_region *mr, 60 struct address_space *mapping, 61 unsigned int max_segment) 62 { 63 const unsigned long page_count = size / PAGE_SIZE; 64 unsigned long i; 65 struct scatterlist *sg; 66 struct page *page; 67 unsigned long last_pfn = 0; /* suppress gcc warning */ 68 gfp_t noreclaim; 69 int ret; 70 71 /* 72 * If there's no chance of allocating enough pages for the whole 73 * object, bail early. 74 */ 75 if (size > resource_size(&mr->region)) 76 return -ENOMEM; 77 78 if (sg_alloc_table(st, page_count, GFP_KERNEL)) 79 return -ENOMEM; 80 81 /* 82 * Get the list of pages out of our struct file. They'll be pinned 83 * at this point until we release them. 84 * 85 * Fail silently without starting the shrinker 86 */ 87 mapping_set_unevictable(mapping); 88 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 89 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 90 91 sg = st->sgl; 92 st->nents = 0; 93 for (i = 0; i < page_count; i++) { 94 const unsigned int shrink[] = { 95 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 96 0, 97 }, *s = shrink; 98 gfp_t gfp = noreclaim; 99 100 do { 101 cond_resched(); 102 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 103 if (!IS_ERR(page)) 104 break; 105 106 if (!*s) { 107 ret = PTR_ERR(page); 108 goto err_sg; 109 } 110 111 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++); 112 113 /* 114 * We've tried hard to allocate the memory by reaping 115 * our own buffer, now let the real VM do its job and 116 * go down in flames if truly OOM. 117 * 118 * However, since graphics tend to be disposable, 119 * defer the oom here by reporting the ENOMEM back 120 * to userspace. 121 */ 122 if (!*s) { 123 /* reclaim and warn, but no oom */ 124 gfp = mapping_gfp_mask(mapping); 125 126 /* 127 * Our bo are always dirty and so we require 128 * kswapd to reclaim our pages (direct reclaim 129 * does not effectively begin pageout of our 130 * buffers on its own). However, direct reclaim 131 * only waits for kswapd when under allocation 132 * congestion. So as a result __GFP_RECLAIM is 133 * unreliable and fails to actually reclaim our 134 * dirty pages -- unless you try over and over 135 * again with !__GFP_NORETRY. However, we still 136 * want to fail this allocation rather than 137 * trigger the out-of-memory killer and for 138 * this we want __GFP_RETRY_MAYFAIL. 139 */ 140 gfp |= __GFP_RETRY_MAYFAIL; 141 } 142 } while (1); 143 144 if (!i || 145 sg->length >= max_segment || 146 page_to_pfn(page) != last_pfn + 1) { 147 if (i) 148 sg = sg_next(sg); 149 150 st->nents++; 151 sg_set_page(sg, page, PAGE_SIZE, 0); 152 } else { 153 sg->length += PAGE_SIZE; 154 } 155 last_pfn = page_to_pfn(page); 156 157 /* Check that the i965g/gm workaround works. */ 158 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL); 159 } 160 if (sg) /* loop terminated early; short sg table */ 161 sg_mark_end(sg); 162 163 /* Trim unused sg entries to avoid wasting memory. */ 164 i915_sg_trim(st); 165 166 return 0; 167 err_sg: 168 sg_mark_end(sg); 169 if (sg != st->sgl) { 170 shmem_sg_free_table(st, mapping, false, false); 171 } else { 172 mapping_clear_unevictable(mapping); 173 sg_free_table(st); 174 } 175 176 /* 177 * shmemfs first checks if there is enough memory to allocate the page 178 * and reports ENOSPC should there be insufficient, along with the usual 179 * ENOMEM for a genuine allocation failure. 180 * 181 * We use ENOSPC in our driver to mean that we have run out of aperture 182 * space and so want to translate the error from shmemfs back to our 183 * usual understanding of ENOMEM. 184 */ 185 if (ret == -ENOSPC) 186 ret = -ENOMEM; 187 188 return ret; 189 } 190 191 static int shmem_get_pages(struct drm_i915_gem_object *obj) 192 { 193 struct drm_i915_private *i915 = to_i915(obj->base.dev); 194 struct intel_memory_region *mem = obj->mm.region; 195 struct address_space *mapping = obj->base.filp->f_mapping; 196 const unsigned long page_count = obj->base.size / PAGE_SIZE; 197 unsigned int max_segment = i915_sg_segment_size(); 198 struct sg_table *st; 199 struct sgt_iter sgt_iter; 200 struct page *page; 201 int ret; 202 203 /* 204 * Assert that the object is not currently in any GPU domain. As it 205 * wasn't in the GTT, there shouldn't be any way it could have been in 206 * a GPU cache 207 */ 208 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 209 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 210 211 rebuild_st: 212 st = kmalloc(sizeof(*st), GFP_KERNEL); 213 if (!st) 214 return -ENOMEM; 215 216 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping, 217 max_segment); 218 if (ret) 219 goto err_st; 220 221 ret = i915_gem_gtt_prepare_pages(obj, st); 222 if (ret) { 223 /* 224 * DMA remapping failed? One possible cause is that 225 * it could not reserve enough large entries, asking 226 * for PAGE_SIZE chunks instead may be helpful. 227 */ 228 if (max_segment > PAGE_SIZE) { 229 for_each_sgt_page(page, sgt_iter, st) 230 put_page(page); 231 sg_free_table(st); 232 kfree(st); 233 234 max_segment = PAGE_SIZE; 235 goto rebuild_st; 236 } else { 237 dev_warn(i915->drm.dev, 238 "Failed to DMA remap %lu pages\n", 239 page_count); 240 goto err_pages; 241 } 242 } 243 244 if (i915_gem_object_needs_bit17_swizzle(obj)) 245 i915_gem_object_do_bit_17_swizzle(obj, st); 246 247 if (i915_gem_object_can_bypass_llc(obj)) 248 obj->cache_dirty = true; 249 250 __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl)); 251 252 return 0; 253 254 err_pages: 255 shmem_sg_free_table(st, mapping, false, false); 256 /* 257 * shmemfs first checks if there is enough memory to allocate the page 258 * and reports ENOSPC should there be insufficient, along with the usual 259 * ENOMEM for a genuine allocation failure. 260 * 261 * We use ENOSPC in our driver to mean that we have run out of aperture 262 * space and so want to translate the error from shmemfs back to our 263 * usual understanding of ENOMEM. 264 */ 265 err_st: 266 if (ret == -ENOSPC) 267 ret = -ENOMEM; 268 269 kfree(st); 270 271 return ret; 272 } 273 274 static int 275 shmem_truncate(struct drm_i915_gem_object *obj) 276 { 277 /* 278 * Our goal here is to return as much of the memory as 279 * is possible back to the system as we are called from OOM. 280 * To do this we must instruct the shmfs to drop all of its 281 * backing pages, *now*. 282 */ 283 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 284 obj->mm.madv = __I915_MADV_PURGED; 285 obj->mm.pages = ERR_PTR(-EFAULT); 286 287 return 0; 288 } 289 290 void __shmem_writeback(size_t size, struct address_space *mapping) 291 { 292 struct writeback_control wbc = { 293 .sync_mode = WB_SYNC_NONE, 294 .nr_to_write = SWAP_CLUSTER_MAX, 295 .range_start = 0, 296 .range_end = LLONG_MAX, 297 .for_reclaim = 1, 298 }; 299 unsigned long i; 300 301 /* 302 * Leave mmapings intact (GTT will have been revoked on unbinding, 303 * leaving only CPU mmapings around) and add those pages to the LRU 304 * instead of invoking writeback so they are aged and paged out 305 * as normal. 306 */ 307 308 /* Begin writeback on each dirty page */ 309 for (i = 0; i < size >> PAGE_SHIFT; i++) { 310 struct page *page; 311 312 page = find_lock_page(mapping, i); 313 if (!page) 314 continue; 315 316 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 317 int ret; 318 319 SetPageReclaim(page); 320 ret = mapping->a_ops->writepage(page, &wbc); 321 if (!PageWriteback(page)) 322 ClearPageReclaim(page); 323 if (!ret) 324 goto put; 325 } 326 unlock_page(page); 327 put: 328 put_page(page); 329 } 330 } 331 332 static void 333 shmem_writeback(struct drm_i915_gem_object *obj) 334 { 335 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping); 336 } 337 338 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags) 339 { 340 switch (obj->mm.madv) { 341 case I915_MADV_DONTNEED: 342 return i915_gem_object_truncate(obj); 343 case __I915_MADV_PURGED: 344 return 0; 345 } 346 347 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) 348 shmem_writeback(obj); 349 350 return 0; 351 } 352 353 void 354 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 355 struct sg_table *pages, 356 bool needs_clflush) 357 { 358 struct drm_i915_private *i915 = to_i915(obj->base.dev); 359 360 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 361 362 if (obj->mm.madv == I915_MADV_DONTNEED) 363 obj->mm.dirty = false; 364 365 if (needs_clflush && 366 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 367 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 368 drm_clflush_sg(pages); 369 370 __start_cpu_write(obj); 371 /* 372 * On non-LLC platforms, force the flush-on-acquire if this is ever 373 * swapped-in. Our async flush path is not trust worthy enough yet(and 374 * happens in the wrong order), and with some tricks it's conceivable 375 * for userspace to change the cache-level to I915_CACHE_NONE after the 376 * pages are swapped-in, and since execbuf binds the object before doing 377 * the async flush, we have a race window. 378 */ 379 if (!HAS_LLC(i915)) 380 obj->cache_dirty = true; 381 } 382 383 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) 384 { 385 __i915_gem_object_release_shmem(obj, pages, true); 386 387 i915_gem_gtt_finish_pages(obj, pages); 388 389 if (i915_gem_object_needs_bit17_swizzle(obj)) 390 i915_gem_object_save_bit_17_swizzle(obj, pages); 391 392 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping, 393 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED); 394 kfree(pages); 395 obj->mm.dirty = false; 396 } 397 398 static void 399 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 400 { 401 if (likely(i915_gem_object_has_struct_page(obj))) 402 i915_gem_object_put_pages_shmem(obj, pages); 403 else 404 i915_gem_object_put_pages_phys(obj, pages); 405 } 406 407 static int 408 shmem_pwrite(struct drm_i915_gem_object *obj, 409 const struct drm_i915_gem_pwrite *arg) 410 { 411 struct address_space *mapping = obj->base.filp->f_mapping; 412 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 413 u64 remain, offset; 414 unsigned int pg; 415 416 /* Caller already validated user args */ 417 GEM_BUG_ON(!access_ok(user_data, arg->size)); 418 419 if (!i915_gem_object_has_struct_page(obj)) 420 return i915_gem_object_pwrite_phys(obj, arg); 421 422 /* 423 * Before we instantiate/pin the backing store for our use, we 424 * can prepopulate the shmemfs filp efficiently using a write into 425 * the pagecache. We avoid the penalty of instantiating all the 426 * pages, important if the user is just writing to a few and never 427 * uses the object on the GPU, and using a direct write into shmemfs 428 * allows it to avoid the cost of retrieving a page (either swapin 429 * or clearing-before-use) before it is overwritten. 430 */ 431 if (i915_gem_object_has_pages(obj)) 432 return -ENODEV; 433 434 if (obj->mm.madv != I915_MADV_WILLNEED) 435 return -EFAULT; 436 437 /* 438 * Before the pages are instantiated the object is treated as being 439 * in the CPU domain. The pages will be clflushed as required before 440 * use, and we can freely write into the pages directly. If userspace 441 * races pwrite with any other operation; corruption will ensue - 442 * that is userspace's prerogative! 443 */ 444 445 remain = arg->size; 446 offset = arg->offset; 447 pg = offset_in_page(offset); 448 449 do { 450 unsigned int len, unwritten; 451 struct page *page; 452 void *data, *vaddr; 453 int err; 454 char c; 455 456 len = PAGE_SIZE - pg; 457 if (len > remain) 458 len = remain; 459 460 /* Prefault the user page to reduce potential recursion */ 461 err = __get_user(c, user_data); 462 if (err) 463 return err; 464 465 err = __get_user(c, user_data + len - 1); 466 if (err) 467 return err; 468 469 err = pagecache_write_begin(obj->base.filp, mapping, 470 offset, len, 0, 471 &page, &data); 472 if (err < 0) 473 return err; 474 475 vaddr = kmap_atomic(page); 476 unwritten = __copy_from_user_inatomic(vaddr + pg, 477 user_data, 478 len); 479 kunmap_atomic(vaddr); 480 481 err = pagecache_write_end(obj->base.filp, mapping, 482 offset, len, len - unwritten, 483 page, data); 484 if (err < 0) 485 return err; 486 487 /* We don't handle -EFAULT, leave it to the caller to check */ 488 if (unwritten) 489 return -ENODEV; 490 491 remain -= len; 492 user_data += len; 493 offset += len; 494 pg = 0; 495 } while (remain); 496 497 return 0; 498 } 499 500 static int 501 shmem_pread(struct drm_i915_gem_object *obj, 502 const struct drm_i915_gem_pread *arg) 503 { 504 if (!i915_gem_object_has_struct_page(obj)) 505 return i915_gem_object_pread_phys(obj, arg); 506 507 return -ENODEV; 508 } 509 510 static void shmem_release(struct drm_i915_gem_object *obj) 511 { 512 if (i915_gem_object_has_struct_page(obj)) 513 i915_gem_object_release_memory_region(obj); 514 515 fput(obj->base.filp); 516 } 517 518 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 519 .name = "i915_gem_object_shmem", 520 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 521 522 .get_pages = shmem_get_pages, 523 .put_pages = shmem_put_pages, 524 .truncate = shmem_truncate, 525 .shrink = shmem_shrink, 526 527 .pwrite = shmem_pwrite, 528 .pread = shmem_pread, 529 530 .release = shmem_release, 531 }; 532 533 static int __create_shmem(struct drm_i915_private *i915, 534 struct drm_gem_object *obj, 535 resource_size_t size) 536 { 537 unsigned long flags = VM_NORESERVE; 538 struct file *filp; 539 540 drm_gem_private_object_init(&i915->drm, obj, size); 541 542 if (i915->mm.gemfs) 543 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 544 flags); 545 else 546 filp = shmem_file_setup("i915", size, flags); 547 if (IS_ERR(filp)) 548 return PTR_ERR(filp); 549 550 obj->filp = filp; 551 return 0; 552 } 553 554 static int shmem_object_init(struct intel_memory_region *mem, 555 struct drm_i915_gem_object *obj, 556 resource_size_t offset, 557 resource_size_t size, 558 resource_size_t page_size, 559 unsigned int flags) 560 { 561 static struct lock_class_key lock_class; 562 struct drm_i915_private *i915 = mem->i915; 563 struct address_space *mapping; 564 unsigned int cache_level; 565 gfp_t mask; 566 int ret; 567 568 ret = __create_shmem(i915, &obj->base, size); 569 if (ret) 570 return ret; 571 572 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 573 if (IS_I965GM(i915) || IS_I965G(i915)) { 574 /* 965gm cannot relocate objects above 4GiB. */ 575 mask &= ~__GFP_HIGHMEM; 576 mask |= __GFP_DMA32; 577 } 578 579 mapping = obj->base.filp->f_mapping; 580 mapping_set_gfp_mask(mapping, mask); 581 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 582 583 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0); 584 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; 585 obj->write_domain = I915_GEM_DOMAIN_CPU; 586 obj->read_domains = I915_GEM_DOMAIN_CPU; 587 588 if (HAS_LLC(i915)) 589 /* On some devices, we can have the GPU use the LLC (the CPU 590 * cache) for about a 10% performance improvement 591 * compared to uncached. Graphics requests other than 592 * display scanout are coherent with the CPU in 593 * accessing this cache. This means in this mode we 594 * don't need to clflush on the CPU side, and on the 595 * GPU side we only need to flush internal caches to 596 * get data visible to the CPU. 597 * 598 * However, we maintain the display planes as UC, and so 599 * need to rebind when first used as such. 600 */ 601 cache_level = I915_CACHE_LLC; 602 else 603 cache_level = I915_CACHE_NONE; 604 605 i915_gem_object_set_cache_coherency(obj, cache_level); 606 607 i915_gem_object_init_memory_region(obj, mem); 608 609 return 0; 610 } 611 612 struct drm_i915_gem_object * 613 i915_gem_object_create_shmem(struct drm_i915_private *i915, 614 resource_size_t size) 615 { 616 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 617 size, 0, 0); 618 } 619 620 /* Allocate a new GEM object and fill it with the supplied data */ 621 struct drm_i915_gem_object * 622 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 623 const void *data, resource_size_t size) 624 { 625 struct drm_i915_gem_object *obj; 626 struct file *file; 627 resource_size_t offset; 628 int err; 629 630 GEM_WARN_ON(IS_DGFX(dev_priv)); 631 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 632 if (IS_ERR(obj)) 633 return obj; 634 635 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 636 637 file = obj->base.filp; 638 offset = 0; 639 do { 640 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 641 struct page *page; 642 void *pgdata, *vaddr; 643 644 err = pagecache_write_begin(file, file->f_mapping, 645 offset, len, 0, 646 &page, &pgdata); 647 if (err < 0) 648 goto fail; 649 650 vaddr = kmap(page); 651 memcpy(vaddr, data, len); 652 kunmap(page); 653 654 err = pagecache_write_end(file, file->f_mapping, 655 offset, len, len, 656 page, pgdata); 657 if (err < 0) 658 goto fail; 659 660 size -= len; 661 data += len; 662 offset += len; 663 } while (size); 664 665 return obj; 666 667 fail: 668 i915_gem_object_put(obj); 669 return ERR_PTR(err); 670 } 671 672 static int init_shmem(struct intel_memory_region *mem) 673 { 674 int err; 675 676 err = i915_gemfs_init(mem->i915); 677 if (err) { 678 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", 679 err); 680 } 681 682 intel_memory_region_set_name(mem, "system"); 683 684 return 0; /* Don't error, we can simply fallback to the kernel mnt */ 685 } 686 687 static int release_shmem(struct intel_memory_region *mem) 688 { 689 i915_gemfs_fini(mem->i915); 690 return 0; 691 } 692 693 static const struct intel_memory_region_ops shmem_region_ops = { 694 .init = init_shmem, 695 .release = release_shmem, 696 .init_object = shmem_object_init, 697 }; 698 699 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, 700 u16 type, u16 instance) 701 { 702 return intel_memory_region_create(i915, 0, 703 totalram_pages() << PAGE_SHIFT, 704 PAGE_SIZE, 0, 0, 705 type, instance, 706 &shmem_region_ops); 707 } 708 709 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj) 710 { 711 return obj->ops == &i915_gem_shmem_ops; 712 } 713