1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/swap.h> 9 10 #include "gem/i915_gem_region.h" 11 #include "i915_drv.h" 12 #include "i915_gemfs.h" 13 #include "i915_gem_object.h" 14 #include "i915_scatterlist.h" 15 #include "i915_trace.h" 16 17 /* 18 * Move pages to appropriate lru and release the pagevec, decrementing the 19 * ref count of those pages. 20 */ 21 static void check_release_pagevec(struct pagevec *pvec) 22 { 23 check_move_unevictable_pages(pvec); 24 __pagevec_release(pvec); 25 cond_resched(); 26 } 27 28 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 29 bool dirty, bool backup) 30 { 31 struct sgt_iter sgt_iter; 32 struct pagevec pvec; 33 struct page *page; 34 35 mapping_clear_unevictable(mapping); 36 37 pagevec_init(&pvec); 38 for_each_sgt_page(page, sgt_iter, st) { 39 if (dirty) 40 set_page_dirty(page); 41 42 if (backup) 43 mark_page_accessed(page); 44 45 if (!pagevec_add(&pvec, page)) 46 check_release_pagevec(&pvec); 47 } 48 if (pagevec_count(&pvec)) 49 check_release_pagevec(&pvec); 50 51 sg_free_table(st); 52 } 53 54 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 55 size_t size, struct intel_memory_region *mr, 56 struct address_space *mapping, 57 unsigned int max_segment) 58 { 59 const unsigned long page_count = size / PAGE_SIZE; 60 unsigned long i; 61 struct scatterlist *sg; 62 struct page *page; 63 unsigned long last_pfn = 0; /* suppress gcc warning */ 64 gfp_t noreclaim; 65 int ret; 66 67 /* 68 * If there's no chance of allocating enough pages for the whole 69 * object, bail early. 70 */ 71 if (size > resource_size(&mr->region)) 72 return -ENOMEM; 73 74 if (sg_alloc_table(st, page_count, GFP_KERNEL)) 75 return -ENOMEM; 76 77 /* 78 * Get the list of pages out of our struct file. They'll be pinned 79 * at this point until we release them. 80 * 81 * Fail silently without starting the shrinker 82 */ 83 mapping_set_unevictable(mapping); 84 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 85 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 86 87 sg = st->sgl; 88 st->nents = 0; 89 for (i = 0; i < page_count; i++) { 90 const unsigned int shrink[] = { 91 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 92 0, 93 }, *s = shrink; 94 gfp_t gfp = noreclaim; 95 96 do { 97 cond_resched(); 98 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 99 if (!IS_ERR(page)) 100 break; 101 102 if (!*s) { 103 ret = PTR_ERR(page); 104 goto err_sg; 105 } 106 107 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++); 108 109 /* 110 * We've tried hard to allocate the memory by reaping 111 * our own buffer, now let the real VM do its job and 112 * go down in flames if truly OOM. 113 * 114 * However, since graphics tend to be disposable, 115 * defer the oom here by reporting the ENOMEM back 116 * to userspace. 117 */ 118 if (!*s) { 119 /* reclaim and warn, but no oom */ 120 gfp = mapping_gfp_mask(mapping); 121 122 /* 123 * Our bo are always dirty and so we require 124 * kswapd to reclaim our pages (direct reclaim 125 * does not effectively begin pageout of our 126 * buffers on its own). However, direct reclaim 127 * only waits for kswapd when under allocation 128 * congestion. So as a result __GFP_RECLAIM is 129 * unreliable and fails to actually reclaim our 130 * dirty pages -- unless you try over and over 131 * again with !__GFP_NORETRY. However, we still 132 * want to fail this allocation rather than 133 * trigger the out-of-memory killer and for 134 * this we want __GFP_RETRY_MAYFAIL. 135 */ 136 gfp |= __GFP_RETRY_MAYFAIL; 137 } 138 } while (1); 139 140 if (!i || 141 sg->length >= max_segment || 142 page_to_pfn(page) != last_pfn + 1) { 143 if (i) 144 sg = sg_next(sg); 145 146 st->nents++; 147 sg_set_page(sg, page, PAGE_SIZE, 0); 148 } else { 149 sg->length += PAGE_SIZE; 150 } 151 last_pfn = page_to_pfn(page); 152 153 /* Check that the i965g/gm workaround works. */ 154 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL); 155 } 156 if (sg) /* loop terminated early; short sg table */ 157 sg_mark_end(sg); 158 159 /* Trim unused sg entries to avoid wasting memory. */ 160 i915_sg_trim(st); 161 162 return 0; 163 err_sg: 164 sg_mark_end(sg); 165 if (sg != st->sgl) { 166 shmem_sg_free_table(st, mapping, false, false); 167 } else { 168 mapping_clear_unevictable(mapping); 169 sg_free_table(st); 170 } 171 172 /* 173 * shmemfs first checks if there is enough memory to allocate the page 174 * and reports ENOSPC should there be insufficient, along with the usual 175 * ENOMEM for a genuine allocation failure. 176 * 177 * We use ENOSPC in our driver to mean that we have run out of aperture 178 * space and so want to translate the error from shmemfs back to our 179 * usual understanding of ENOMEM. 180 */ 181 if (ret == -ENOSPC) 182 ret = -ENOMEM; 183 184 return ret; 185 } 186 187 static int shmem_get_pages(struct drm_i915_gem_object *obj) 188 { 189 struct drm_i915_private *i915 = to_i915(obj->base.dev); 190 struct intel_memory_region *mem = obj->mm.region; 191 struct address_space *mapping = obj->base.filp->f_mapping; 192 const unsigned long page_count = obj->base.size / PAGE_SIZE; 193 unsigned int max_segment = i915_sg_segment_size(); 194 struct sg_table *st; 195 struct sgt_iter sgt_iter; 196 struct page *page; 197 int ret; 198 199 /* 200 * Assert that the object is not currently in any GPU domain. As it 201 * wasn't in the GTT, there shouldn't be any way it could have been in 202 * a GPU cache 203 */ 204 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 205 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 206 207 rebuild_st: 208 st = kmalloc(sizeof(*st), GFP_KERNEL); 209 if (!st) 210 return -ENOMEM; 211 212 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping, 213 max_segment); 214 if (ret) 215 goto err_st; 216 217 ret = i915_gem_gtt_prepare_pages(obj, st); 218 if (ret) { 219 /* 220 * DMA remapping failed? One possible cause is that 221 * it could not reserve enough large entries, asking 222 * for PAGE_SIZE chunks instead may be helpful. 223 */ 224 if (max_segment > PAGE_SIZE) { 225 for_each_sgt_page(page, sgt_iter, st) 226 put_page(page); 227 sg_free_table(st); 228 kfree(st); 229 230 max_segment = PAGE_SIZE; 231 goto rebuild_st; 232 } else { 233 dev_warn(i915->drm.dev, 234 "Failed to DMA remap %lu pages\n", 235 page_count); 236 goto err_pages; 237 } 238 } 239 240 if (i915_gem_object_needs_bit17_swizzle(obj)) 241 i915_gem_object_do_bit_17_swizzle(obj, st); 242 243 if (i915_gem_object_can_bypass_llc(obj)) 244 obj->cache_dirty = true; 245 246 __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl)); 247 248 return 0; 249 250 err_pages: 251 shmem_sg_free_table(st, mapping, false, false); 252 /* 253 * shmemfs first checks if there is enough memory to allocate the page 254 * and reports ENOSPC should there be insufficient, along with the usual 255 * ENOMEM for a genuine allocation failure. 256 * 257 * We use ENOSPC in our driver to mean that we have run out of aperture 258 * space and so want to translate the error from shmemfs back to our 259 * usual understanding of ENOMEM. 260 */ 261 err_st: 262 if (ret == -ENOSPC) 263 ret = -ENOMEM; 264 265 kfree(st); 266 267 return ret; 268 } 269 270 static int 271 shmem_truncate(struct drm_i915_gem_object *obj) 272 { 273 /* 274 * Our goal here is to return as much of the memory as 275 * is possible back to the system as we are called from OOM. 276 * To do this we must instruct the shmfs to drop all of its 277 * backing pages, *now*. 278 */ 279 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 280 obj->mm.madv = __I915_MADV_PURGED; 281 obj->mm.pages = ERR_PTR(-EFAULT); 282 283 return 0; 284 } 285 286 void __shmem_writeback(size_t size, struct address_space *mapping) 287 { 288 struct writeback_control wbc = { 289 .sync_mode = WB_SYNC_NONE, 290 .nr_to_write = SWAP_CLUSTER_MAX, 291 .range_start = 0, 292 .range_end = LLONG_MAX, 293 .for_reclaim = 1, 294 }; 295 unsigned long i; 296 297 /* 298 * Leave mmapings intact (GTT will have been revoked on unbinding, 299 * leaving only CPU mmapings around) and add those pages to the LRU 300 * instead of invoking writeback so they are aged and paged out 301 * as normal. 302 */ 303 304 /* Begin writeback on each dirty page */ 305 for (i = 0; i < size >> PAGE_SHIFT; i++) { 306 struct page *page; 307 308 page = find_lock_page(mapping, i); 309 if (!page) 310 continue; 311 312 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 313 int ret; 314 315 SetPageReclaim(page); 316 ret = mapping->a_ops->writepage(page, &wbc); 317 if (!PageWriteback(page)) 318 ClearPageReclaim(page); 319 if (!ret) 320 goto put; 321 } 322 unlock_page(page); 323 put: 324 put_page(page); 325 } 326 } 327 328 static void 329 shmem_writeback(struct drm_i915_gem_object *obj) 330 { 331 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping); 332 } 333 334 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags) 335 { 336 switch (obj->mm.madv) { 337 case I915_MADV_DONTNEED: 338 return i915_gem_object_truncate(obj); 339 case __I915_MADV_PURGED: 340 return 0; 341 } 342 343 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) 344 shmem_writeback(obj); 345 346 return 0; 347 } 348 349 void 350 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 351 struct sg_table *pages, 352 bool needs_clflush) 353 { 354 struct drm_i915_private *i915 = to_i915(obj->base.dev); 355 356 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 357 358 if (obj->mm.madv == I915_MADV_DONTNEED) 359 obj->mm.dirty = false; 360 361 if (needs_clflush && 362 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 363 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 364 drm_clflush_sg(pages); 365 366 __start_cpu_write(obj); 367 /* 368 * On non-LLC platforms, force the flush-on-acquire if this is ever 369 * swapped-in. Our async flush path is not trust worthy enough yet(and 370 * happens in the wrong order), and with some tricks it's conceivable 371 * for userspace to change the cache-level to I915_CACHE_NONE after the 372 * pages are swapped-in, and since execbuf binds the object before doing 373 * the async flush, we have a race window. 374 */ 375 if (!HAS_LLC(i915)) 376 obj->cache_dirty = true; 377 } 378 379 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) 380 { 381 __i915_gem_object_release_shmem(obj, pages, true); 382 383 i915_gem_gtt_finish_pages(obj, pages); 384 385 if (i915_gem_object_needs_bit17_swizzle(obj)) 386 i915_gem_object_save_bit_17_swizzle(obj, pages); 387 388 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping, 389 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED); 390 kfree(pages); 391 obj->mm.dirty = false; 392 } 393 394 static void 395 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 396 { 397 if (likely(i915_gem_object_has_struct_page(obj))) 398 i915_gem_object_put_pages_shmem(obj, pages); 399 else 400 i915_gem_object_put_pages_phys(obj, pages); 401 } 402 403 static int 404 shmem_pwrite(struct drm_i915_gem_object *obj, 405 const struct drm_i915_gem_pwrite *arg) 406 { 407 struct address_space *mapping = obj->base.filp->f_mapping; 408 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 409 u64 remain, offset; 410 unsigned int pg; 411 412 /* Caller already validated user args */ 413 GEM_BUG_ON(!access_ok(user_data, arg->size)); 414 415 if (!i915_gem_object_has_struct_page(obj)) 416 return i915_gem_object_pwrite_phys(obj, arg); 417 418 /* 419 * Before we instantiate/pin the backing store for our use, we 420 * can prepopulate the shmemfs filp efficiently using a write into 421 * the pagecache. We avoid the penalty of instantiating all the 422 * pages, important if the user is just writing to a few and never 423 * uses the object on the GPU, and using a direct write into shmemfs 424 * allows it to avoid the cost of retrieving a page (either swapin 425 * or clearing-before-use) before it is overwritten. 426 */ 427 if (i915_gem_object_has_pages(obj)) 428 return -ENODEV; 429 430 if (obj->mm.madv != I915_MADV_WILLNEED) 431 return -EFAULT; 432 433 /* 434 * Before the pages are instantiated the object is treated as being 435 * in the CPU domain. The pages will be clflushed as required before 436 * use, and we can freely write into the pages directly. If userspace 437 * races pwrite with any other operation; corruption will ensue - 438 * that is userspace's prerogative! 439 */ 440 441 remain = arg->size; 442 offset = arg->offset; 443 pg = offset_in_page(offset); 444 445 do { 446 unsigned int len, unwritten; 447 struct page *page; 448 void *data, *vaddr; 449 int err; 450 char c; 451 452 len = PAGE_SIZE - pg; 453 if (len > remain) 454 len = remain; 455 456 /* Prefault the user page to reduce potential recursion */ 457 err = __get_user(c, user_data); 458 if (err) 459 return err; 460 461 err = __get_user(c, user_data + len - 1); 462 if (err) 463 return err; 464 465 err = pagecache_write_begin(obj->base.filp, mapping, 466 offset, len, 0, 467 &page, &data); 468 if (err < 0) 469 return err; 470 471 vaddr = kmap_atomic(page); 472 unwritten = __copy_from_user_inatomic(vaddr + pg, 473 user_data, 474 len); 475 kunmap_atomic(vaddr); 476 477 err = pagecache_write_end(obj->base.filp, mapping, 478 offset, len, len - unwritten, 479 page, data); 480 if (err < 0) 481 return err; 482 483 /* We don't handle -EFAULT, leave it to the caller to check */ 484 if (unwritten) 485 return -ENODEV; 486 487 remain -= len; 488 user_data += len; 489 offset += len; 490 pg = 0; 491 } while (remain); 492 493 return 0; 494 } 495 496 static int 497 shmem_pread(struct drm_i915_gem_object *obj, 498 const struct drm_i915_gem_pread *arg) 499 { 500 if (!i915_gem_object_has_struct_page(obj)) 501 return i915_gem_object_pread_phys(obj, arg); 502 503 return -ENODEV; 504 } 505 506 static void shmem_release(struct drm_i915_gem_object *obj) 507 { 508 if (i915_gem_object_has_struct_page(obj)) 509 i915_gem_object_release_memory_region(obj); 510 511 fput(obj->base.filp); 512 } 513 514 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 515 .name = "i915_gem_object_shmem", 516 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 517 518 .get_pages = shmem_get_pages, 519 .put_pages = shmem_put_pages, 520 .truncate = shmem_truncate, 521 .shrink = shmem_shrink, 522 523 .pwrite = shmem_pwrite, 524 .pread = shmem_pread, 525 526 .release = shmem_release, 527 }; 528 529 static int __create_shmem(struct drm_i915_private *i915, 530 struct drm_gem_object *obj, 531 resource_size_t size) 532 { 533 unsigned long flags = VM_NORESERVE; 534 struct file *filp; 535 536 drm_gem_private_object_init(&i915->drm, obj, size); 537 538 if (i915->mm.gemfs) 539 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 540 flags); 541 else 542 filp = shmem_file_setup("i915", size, flags); 543 if (IS_ERR(filp)) 544 return PTR_ERR(filp); 545 546 obj->filp = filp; 547 return 0; 548 } 549 550 static int shmem_object_init(struct intel_memory_region *mem, 551 struct drm_i915_gem_object *obj, 552 resource_size_t size, 553 resource_size_t page_size, 554 unsigned int flags) 555 { 556 static struct lock_class_key lock_class; 557 struct drm_i915_private *i915 = mem->i915; 558 struct address_space *mapping; 559 unsigned int cache_level; 560 gfp_t mask; 561 int ret; 562 563 ret = __create_shmem(i915, &obj->base, size); 564 if (ret) 565 return ret; 566 567 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 568 if (IS_I965GM(i915) || IS_I965G(i915)) { 569 /* 965gm cannot relocate objects above 4GiB. */ 570 mask &= ~__GFP_HIGHMEM; 571 mask |= __GFP_DMA32; 572 } 573 574 mapping = obj->base.filp->f_mapping; 575 mapping_set_gfp_mask(mapping, mask); 576 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 577 578 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0); 579 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; 580 obj->write_domain = I915_GEM_DOMAIN_CPU; 581 obj->read_domains = I915_GEM_DOMAIN_CPU; 582 583 if (HAS_LLC(i915)) 584 /* On some devices, we can have the GPU use the LLC (the CPU 585 * cache) for about a 10% performance improvement 586 * compared to uncached. Graphics requests other than 587 * display scanout are coherent with the CPU in 588 * accessing this cache. This means in this mode we 589 * don't need to clflush on the CPU side, and on the 590 * GPU side we only need to flush internal caches to 591 * get data visible to the CPU. 592 * 593 * However, we maintain the display planes as UC, and so 594 * need to rebind when first used as such. 595 */ 596 cache_level = I915_CACHE_LLC; 597 else 598 cache_level = I915_CACHE_NONE; 599 600 i915_gem_object_set_cache_coherency(obj, cache_level); 601 602 i915_gem_object_init_memory_region(obj, mem); 603 604 return 0; 605 } 606 607 struct drm_i915_gem_object * 608 i915_gem_object_create_shmem(struct drm_i915_private *i915, 609 resource_size_t size) 610 { 611 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 612 size, 0, 0); 613 } 614 615 /* Allocate a new GEM object and fill it with the supplied data */ 616 struct drm_i915_gem_object * 617 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 618 const void *data, resource_size_t size) 619 { 620 struct drm_i915_gem_object *obj; 621 struct file *file; 622 resource_size_t offset; 623 int err; 624 625 GEM_WARN_ON(IS_DGFX(dev_priv)); 626 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 627 if (IS_ERR(obj)) 628 return obj; 629 630 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 631 632 file = obj->base.filp; 633 offset = 0; 634 do { 635 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 636 struct page *page; 637 void *pgdata, *vaddr; 638 639 err = pagecache_write_begin(file, file->f_mapping, 640 offset, len, 0, 641 &page, &pgdata); 642 if (err < 0) 643 goto fail; 644 645 vaddr = kmap(page); 646 memcpy(vaddr, data, len); 647 kunmap(page); 648 649 err = pagecache_write_end(file, file->f_mapping, 650 offset, len, len, 651 page, pgdata); 652 if (err < 0) 653 goto fail; 654 655 size -= len; 656 data += len; 657 offset += len; 658 } while (size); 659 660 return obj; 661 662 fail: 663 i915_gem_object_put(obj); 664 return ERR_PTR(err); 665 } 666 667 static int init_shmem(struct intel_memory_region *mem) 668 { 669 int err; 670 671 err = i915_gemfs_init(mem->i915); 672 if (err) { 673 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", 674 err); 675 } 676 677 intel_memory_region_set_name(mem, "system"); 678 679 return 0; /* Don't error, we can simply fallback to the kernel mnt */ 680 } 681 682 static int release_shmem(struct intel_memory_region *mem) 683 { 684 i915_gemfs_fini(mem->i915); 685 return 0; 686 } 687 688 static const struct intel_memory_region_ops shmem_region_ops = { 689 .init = init_shmem, 690 .release = release_shmem, 691 .init_object = shmem_object_init, 692 }; 693 694 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, 695 u16 type, u16 instance) 696 { 697 return intel_memory_region_create(i915, 0, 698 totalram_pages() << PAGE_SHIFT, 699 PAGE_SIZE, 0, 700 type, instance, 701 &shmem_region_ops); 702 } 703 704 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj) 705 { 706 return obj->ops == &i915_gem_shmem_ops; 707 } 708