1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/swap.h> 9 10 #include "gem/i915_gem_region.h" 11 #include "i915_drv.h" 12 #include "i915_gemfs.h" 13 #include "i915_gem_object.h" 14 #include "i915_scatterlist.h" 15 #include "i915_trace.h" 16 17 /* 18 * Move pages to appropriate lru and release the pagevec, decrementing the 19 * ref count of those pages. 20 */ 21 static void check_release_pagevec(struct pagevec *pvec) 22 { 23 check_move_unevictable_pages(pvec); 24 __pagevec_release(pvec); 25 cond_resched(); 26 } 27 28 static int shmem_get_pages(struct drm_i915_gem_object *obj) 29 { 30 struct drm_i915_private *i915 = to_i915(obj->base.dev); 31 struct intel_memory_region *mem = obj->mm.region; 32 const unsigned long page_count = obj->base.size / PAGE_SIZE; 33 unsigned long i; 34 struct address_space *mapping; 35 struct sg_table *st; 36 struct scatterlist *sg; 37 struct sgt_iter sgt_iter; 38 struct page *page; 39 unsigned long last_pfn = 0; /* suppress gcc warning */ 40 unsigned int max_segment = i915_sg_segment_size(); 41 unsigned int sg_page_sizes; 42 gfp_t noreclaim; 43 int ret; 44 45 /* 46 * Assert that the object is not currently in any GPU domain. As it 47 * wasn't in the GTT, there shouldn't be any way it could have been in 48 * a GPU cache 49 */ 50 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 51 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 52 53 /* 54 * If there's no chance of allocating enough pages for the whole 55 * object, bail early. 56 */ 57 if (obj->base.size > resource_size(&mem->region)) 58 return -ENOMEM; 59 60 st = kmalloc(sizeof(*st), GFP_KERNEL); 61 if (!st) 62 return -ENOMEM; 63 64 rebuild_st: 65 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 66 kfree(st); 67 return -ENOMEM; 68 } 69 70 /* 71 * Get the list of pages out of our struct file. They'll be pinned 72 * at this point until we release them. 73 * 74 * Fail silently without starting the shrinker 75 */ 76 mapping = obj->base.filp->f_mapping; 77 mapping_set_unevictable(mapping); 78 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 79 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 80 81 sg = st->sgl; 82 st->nents = 0; 83 sg_page_sizes = 0; 84 for (i = 0; i < page_count; i++) { 85 const unsigned int shrink[] = { 86 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 87 0, 88 }, *s = shrink; 89 gfp_t gfp = noreclaim; 90 91 do { 92 cond_resched(); 93 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 94 if (!IS_ERR(page)) 95 break; 96 97 if (!*s) { 98 ret = PTR_ERR(page); 99 goto err_sg; 100 } 101 102 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++); 103 104 /* 105 * We've tried hard to allocate the memory by reaping 106 * our own buffer, now let the real VM do its job and 107 * go down in flames if truly OOM. 108 * 109 * However, since graphics tend to be disposable, 110 * defer the oom here by reporting the ENOMEM back 111 * to userspace. 112 */ 113 if (!*s) { 114 /* reclaim and warn, but no oom */ 115 gfp = mapping_gfp_mask(mapping); 116 117 /* 118 * Our bo are always dirty and so we require 119 * kswapd to reclaim our pages (direct reclaim 120 * does not effectively begin pageout of our 121 * buffers on its own). However, direct reclaim 122 * only waits for kswapd when under allocation 123 * congestion. So as a result __GFP_RECLAIM is 124 * unreliable and fails to actually reclaim our 125 * dirty pages -- unless you try over and over 126 * again with !__GFP_NORETRY. However, we still 127 * want to fail this allocation rather than 128 * trigger the out-of-memory killer and for 129 * this we want __GFP_RETRY_MAYFAIL. 130 */ 131 gfp |= __GFP_RETRY_MAYFAIL; 132 } 133 } while (1); 134 135 if (!i || 136 sg->length >= max_segment || 137 page_to_pfn(page) != last_pfn + 1) { 138 if (i) { 139 sg_page_sizes |= sg->length; 140 sg = sg_next(sg); 141 } 142 st->nents++; 143 sg_set_page(sg, page, PAGE_SIZE, 0); 144 } else { 145 sg->length += PAGE_SIZE; 146 } 147 last_pfn = page_to_pfn(page); 148 149 /* Check that the i965g/gm workaround works. */ 150 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL); 151 } 152 if (sg) { /* loop terminated early; short sg table */ 153 sg_page_sizes |= sg->length; 154 sg_mark_end(sg); 155 } 156 157 /* Trim unused sg entries to avoid wasting memory. */ 158 i915_sg_trim(st); 159 160 ret = i915_gem_gtt_prepare_pages(obj, st); 161 if (ret) { 162 /* 163 * DMA remapping failed? One possible cause is that 164 * it could not reserve enough large entries, asking 165 * for PAGE_SIZE chunks instead may be helpful. 166 */ 167 if (max_segment > PAGE_SIZE) { 168 for_each_sgt_page(page, sgt_iter, st) 169 put_page(page); 170 sg_free_table(st); 171 172 max_segment = PAGE_SIZE; 173 goto rebuild_st; 174 } else { 175 dev_warn(i915->drm.dev, 176 "Failed to DMA remap %lu pages\n", 177 page_count); 178 goto err_pages; 179 } 180 } 181 182 if (i915_gem_object_needs_bit17_swizzle(obj)) 183 i915_gem_object_do_bit_17_swizzle(obj, st); 184 185 if (i915_gem_object_can_bypass_llc(obj)) 186 obj->cache_dirty = true; 187 188 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 189 190 return 0; 191 192 err_sg: 193 sg_mark_end(sg); 194 err_pages: 195 mapping_clear_unevictable(mapping); 196 if (sg != st->sgl) { 197 struct pagevec pvec; 198 199 pagevec_init(&pvec); 200 for_each_sgt_page(page, sgt_iter, st) { 201 if (!pagevec_add(&pvec, page)) 202 check_release_pagevec(&pvec); 203 } 204 if (pagevec_count(&pvec)) 205 check_release_pagevec(&pvec); 206 } 207 sg_free_table(st); 208 kfree(st); 209 210 /* 211 * shmemfs first checks if there is enough memory to allocate the page 212 * and reports ENOSPC should there be insufficient, along with the usual 213 * ENOMEM for a genuine allocation failure. 214 * 215 * We use ENOSPC in our driver to mean that we have run out of aperture 216 * space and so want to translate the error from shmemfs back to our 217 * usual understanding of ENOMEM. 218 */ 219 if (ret == -ENOSPC) 220 ret = -ENOMEM; 221 222 return ret; 223 } 224 225 static void 226 shmem_truncate(struct drm_i915_gem_object *obj) 227 { 228 /* 229 * Our goal here is to return as much of the memory as 230 * is possible back to the system as we are called from OOM. 231 * To do this we must instruct the shmfs to drop all of its 232 * backing pages, *now*. 233 */ 234 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 235 obj->mm.madv = __I915_MADV_PURGED; 236 obj->mm.pages = ERR_PTR(-EFAULT); 237 } 238 239 static void 240 shmem_writeback(struct drm_i915_gem_object *obj) 241 { 242 struct address_space *mapping; 243 struct writeback_control wbc = { 244 .sync_mode = WB_SYNC_NONE, 245 .nr_to_write = SWAP_CLUSTER_MAX, 246 .range_start = 0, 247 .range_end = LLONG_MAX, 248 .for_reclaim = 1, 249 }; 250 unsigned long i; 251 252 /* 253 * Leave mmapings intact (GTT will have been revoked on unbinding, 254 * leaving only CPU mmapings around) and add those pages to the LRU 255 * instead of invoking writeback so they are aged and paged out 256 * as normal. 257 */ 258 mapping = obj->base.filp->f_mapping; 259 260 /* Begin writeback on each dirty page */ 261 for (i = 0; i < obj->base.size >> PAGE_SHIFT; i++) { 262 struct page *page; 263 264 page = find_lock_page(mapping, i); 265 if (!page) 266 continue; 267 268 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 269 int ret; 270 271 SetPageReclaim(page); 272 ret = mapping->a_ops->writepage(page, &wbc); 273 if (!PageWriteback(page)) 274 ClearPageReclaim(page); 275 if (!ret) 276 goto put; 277 } 278 unlock_page(page); 279 put: 280 put_page(page); 281 } 282 } 283 284 void 285 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 286 struct sg_table *pages, 287 bool needs_clflush) 288 { 289 struct drm_i915_private *i915 = to_i915(obj->base.dev); 290 291 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 292 293 if (obj->mm.madv == I915_MADV_DONTNEED) 294 obj->mm.dirty = false; 295 296 if (needs_clflush && 297 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 298 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 299 drm_clflush_sg(pages); 300 301 __start_cpu_write(obj); 302 /* 303 * On non-LLC platforms, force the flush-on-acquire if this is ever 304 * swapped-in. Our async flush path is not trust worthy enough yet(and 305 * happens in the wrong order), and with some tricks it's conceivable 306 * for userspace to change the cache-level to I915_CACHE_NONE after the 307 * pages are swapped-in, and since execbuf binds the object before doing 308 * the async flush, we have a race window. 309 */ 310 if (!HAS_LLC(i915)) 311 obj->cache_dirty = true; 312 } 313 314 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) 315 { 316 struct sgt_iter sgt_iter; 317 struct pagevec pvec; 318 struct page *page; 319 320 GEM_WARN_ON(IS_DGFX(to_i915(obj->base.dev))); 321 __i915_gem_object_release_shmem(obj, pages, true); 322 323 i915_gem_gtt_finish_pages(obj, pages); 324 325 if (i915_gem_object_needs_bit17_swizzle(obj)) 326 i915_gem_object_save_bit_17_swizzle(obj, pages); 327 328 mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping); 329 330 pagevec_init(&pvec); 331 for_each_sgt_page(page, sgt_iter, pages) { 332 if (obj->mm.dirty) 333 set_page_dirty(page); 334 335 if (obj->mm.madv == I915_MADV_WILLNEED) 336 mark_page_accessed(page); 337 338 if (!pagevec_add(&pvec, page)) 339 check_release_pagevec(&pvec); 340 } 341 if (pagevec_count(&pvec)) 342 check_release_pagevec(&pvec); 343 obj->mm.dirty = false; 344 345 sg_free_table(pages); 346 kfree(pages); 347 } 348 349 static void 350 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 351 { 352 if (likely(i915_gem_object_has_struct_page(obj))) 353 i915_gem_object_put_pages_shmem(obj, pages); 354 else 355 i915_gem_object_put_pages_phys(obj, pages); 356 } 357 358 static int 359 shmem_pwrite(struct drm_i915_gem_object *obj, 360 const struct drm_i915_gem_pwrite *arg) 361 { 362 struct address_space *mapping = obj->base.filp->f_mapping; 363 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 364 u64 remain, offset; 365 unsigned int pg; 366 367 /* Caller already validated user args */ 368 GEM_BUG_ON(!access_ok(user_data, arg->size)); 369 370 if (!i915_gem_object_has_struct_page(obj)) 371 return i915_gem_object_pwrite_phys(obj, arg); 372 373 /* 374 * Before we instantiate/pin the backing store for our use, we 375 * can prepopulate the shmemfs filp efficiently using a write into 376 * the pagecache. We avoid the penalty of instantiating all the 377 * pages, important if the user is just writing to a few and never 378 * uses the object on the GPU, and using a direct write into shmemfs 379 * allows it to avoid the cost of retrieving a page (either swapin 380 * or clearing-before-use) before it is overwritten. 381 */ 382 if (i915_gem_object_has_pages(obj)) 383 return -ENODEV; 384 385 if (obj->mm.madv != I915_MADV_WILLNEED) 386 return -EFAULT; 387 388 /* 389 * Before the pages are instantiated the object is treated as being 390 * in the CPU domain. The pages will be clflushed as required before 391 * use, and we can freely write into the pages directly. If userspace 392 * races pwrite with any other operation; corruption will ensue - 393 * that is userspace's prerogative! 394 */ 395 396 remain = arg->size; 397 offset = arg->offset; 398 pg = offset_in_page(offset); 399 400 do { 401 unsigned int len, unwritten; 402 struct page *page; 403 void *data, *vaddr; 404 int err; 405 char c; 406 407 len = PAGE_SIZE - pg; 408 if (len > remain) 409 len = remain; 410 411 /* Prefault the user page to reduce potential recursion */ 412 err = __get_user(c, user_data); 413 if (err) 414 return err; 415 416 err = __get_user(c, user_data + len - 1); 417 if (err) 418 return err; 419 420 err = pagecache_write_begin(obj->base.filp, mapping, 421 offset, len, 0, 422 &page, &data); 423 if (err < 0) 424 return err; 425 426 vaddr = kmap_atomic(page); 427 unwritten = __copy_from_user_inatomic(vaddr + pg, 428 user_data, 429 len); 430 kunmap_atomic(vaddr); 431 432 err = pagecache_write_end(obj->base.filp, mapping, 433 offset, len, len - unwritten, 434 page, data); 435 if (err < 0) 436 return err; 437 438 /* We don't handle -EFAULT, leave it to the caller to check */ 439 if (unwritten) 440 return -ENODEV; 441 442 remain -= len; 443 user_data += len; 444 offset += len; 445 pg = 0; 446 } while (remain); 447 448 return 0; 449 } 450 451 static int 452 shmem_pread(struct drm_i915_gem_object *obj, 453 const struct drm_i915_gem_pread *arg) 454 { 455 if (!i915_gem_object_has_struct_page(obj)) 456 return i915_gem_object_pread_phys(obj, arg); 457 458 return -ENODEV; 459 } 460 461 static void shmem_release(struct drm_i915_gem_object *obj) 462 { 463 if (i915_gem_object_has_struct_page(obj)) 464 i915_gem_object_release_memory_region(obj); 465 466 fput(obj->base.filp); 467 } 468 469 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 470 .name = "i915_gem_object_shmem", 471 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 472 473 .get_pages = shmem_get_pages, 474 .put_pages = shmem_put_pages, 475 .truncate = shmem_truncate, 476 .writeback = shmem_writeback, 477 478 .pwrite = shmem_pwrite, 479 .pread = shmem_pread, 480 481 .release = shmem_release, 482 }; 483 484 static int __create_shmem(struct drm_i915_private *i915, 485 struct drm_gem_object *obj, 486 resource_size_t size) 487 { 488 unsigned long flags = VM_NORESERVE; 489 struct file *filp; 490 491 drm_gem_private_object_init(&i915->drm, obj, size); 492 493 if (i915->mm.gemfs) 494 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 495 flags); 496 else 497 filp = shmem_file_setup("i915", size, flags); 498 if (IS_ERR(filp)) 499 return PTR_ERR(filp); 500 501 obj->filp = filp; 502 return 0; 503 } 504 505 static int shmem_object_init(struct intel_memory_region *mem, 506 struct drm_i915_gem_object *obj, 507 resource_size_t size, 508 resource_size_t page_size, 509 unsigned int flags) 510 { 511 static struct lock_class_key lock_class; 512 struct drm_i915_private *i915 = mem->i915; 513 struct address_space *mapping; 514 unsigned int cache_level; 515 gfp_t mask; 516 int ret; 517 518 ret = __create_shmem(i915, &obj->base, size); 519 if (ret) 520 return ret; 521 522 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 523 if (IS_I965GM(i915) || IS_I965G(i915)) { 524 /* 965gm cannot relocate objects above 4GiB. */ 525 mask &= ~__GFP_HIGHMEM; 526 mask |= __GFP_DMA32; 527 } 528 529 mapping = obj->base.filp->f_mapping; 530 mapping_set_gfp_mask(mapping, mask); 531 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 532 533 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0); 534 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; 535 obj->write_domain = I915_GEM_DOMAIN_CPU; 536 obj->read_domains = I915_GEM_DOMAIN_CPU; 537 538 if (HAS_LLC(i915)) 539 /* On some devices, we can have the GPU use the LLC (the CPU 540 * cache) for about a 10% performance improvement 541 * compared to uncached. Graphics requests other than 542 * display scanout are coherent with the CPU in 543 * accessing this cache. This means in this mode we 544 * don't need to clflush on the CPU side, and on the 545 * GPU side we only need to flush internal caches to 546 * get data visible to the CPU. 547 * 548 * However, we maintain the display planes as UC, and so 549 * need to rebind when first used as such. 550 */ 551 cache_level = I915_CACHE_LLC; 552 else 553 cache_level = I915_CACHE_NONE; 554 555 i915_gem_object_set_cache_coherency(obj, cache_level); 556 557 i915_gem_object_init_memory_region(obj, mem); 558 559 return 0; 560 } 561 562 struct drm_i915_gem_object * 563 i915_gem_object_create_shmem(struct drm_i915_private *i915, 564 resource_size_t size) 565 { 566 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 567 size, 0, 0); 568 } 569 570 /* Allocate a new GEM object and fill it with the supplied data */ 571 struct drm_i915_gem_object * 572 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 573 const void *data, resource_size_t size) 574 { 575 struct drm_i915_gem_object *obj; 576 struct file *file; 577 resource_size_t offset; 578 int err; 579 580 GEM_WARN_ON(IS_DGFX(dev_priv)); 581 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 582 if (IS_ERR(obj)) 583 return obj; 584 585 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 586 587 file = obj->base.filp; 588 offset = 0; 589 do { 590 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 591 struct page *page; 592 void *pgdata, *vaddr; 593 594 err = pagecache_write_begin(file, file->f_mapping, 595 offset, len, 0, 596 &page, &pgdata); 597 if (err < 0) 598 goto fail; 599 600 vaddr = kmap(page); 601 memcpy(vaddr, data, len); 602 kunmap(page); 603 604 err = pagecache_write_end(file, file->f_mapping, 605 offset, len, len, 606 page, pgdata); 607 if (err < 0) 608 goto fail; 609 610 size -= len; 611 data += len; 612 offset += len; 613 } while (size); 614 615 return obj; 616 617 fail: 618 i915_gem_object_put(obj); 619 return ERR_PTR(err); 620 } 621 622 static int init_shmem(struct intel_memory_region *mem) 623 { 624 int err; 625 626 err = i915_gemfs_init(mem->i915); 627 if (err) { 628 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", 629 err); 630 } 631 632 intel_memory_region_set_name(mem, "system"); 633 634 return 0; /* Don't error, we can simply fallback to the kernel mnt */ 635 } 636 637 static void release_shmem(struct intel_memory_region *mem) 638 { 639 i915_gemfs_fini(mem->i915); 640 } 641 642 static const struct intel_memory_region_ops shmem_region_ops = { 643 .init = init_shmem, 644 .release = release_shmem, 645 .init_object = shmem_object_init, 646 }; 647 648 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, 649 u16 type, u16 instance) 650 { 651 return intel_memory_region_create(i915, 0, 652 totalram_pages() << PAGE_SHIFT, 653 PAGE_SIZE, 0, 654 type, instance, 655 &shmem_region_ops); 656 } 657 658 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj) 659 { 660 return obj->ops == &i915_gem_shmem_ops; 661 } 662