1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/swap.h> 9 10 #include "i915_drv.h" 11 #include "i915_gem_object.h" 12 #include "i915_scatterlist.h" 13 #include "i915_trace.h" 14 15 /* 16 * Move pages to appropriate lru and release the pagevec, decrementing the 17 * ref count of those pages. 18 */ 19 static void check_release_pagevec(struct pagevec *pvec) 20 { 21 check_move_unevictable_pages(pvec); 22 __pagevec_release(pvec); 23 cond_resched(); 24 } 25 26 static int shmem_get_pages(struct drm_i915_gem_object *obj) 27 { 28 struct drm_i915_private *i915 = to_i915(obj->base.dev); 29 const unsigned long page_count = obj->base.size / PAGE_SIZE; 30 unsigned long i; 31 struct address_space *mapping; 32 struct sg_table *st; 33 struct scatterlist *sg; 34 struct sgt_iter sgt_iter; 35 struct page *page; 36 unsigned long last_pfn = 0; /* suppress gcc warning */ 37 unsigned int max_segment = i915_sg_segment_size(); 38 unsigned int sg_page_sizes; 39 struct pagevec pvec; 40 gfp_t noreclaim; 41 int ret; 42 43 /* 44 * Assert that the object is not currently in any GPU domain. As it 45 * wasn't in the GTT, there shouldn't be any way it could have been in 46 * a GPU cache 47 */ 48 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 49 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 50 51 /* 52 * If there's no chance of allocating enough pages for the whole 53 * object, bail early. 54 */ 55 if (page_count > totalram_pages()) 56 return -ENOMEM; 57 58 st = kmalloc(sizeof(*st), GFP_KERNEL); 59 if (!st) 60 return -ENOMEM; 61 62 rebuild_st: 63 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 64 kfree(st); 65 return -ENOMEM; 66 } 67 68 /* 69 * Get the list of pages out of our struct file. They'll be pinned 70 * at this point until we release them. 71 * 72 * Fail silently without starting the shrinker 73 */ 74 mapping = obj->base.filp->f_mapping; 75 mapping_set_unevictable(mapping); 76 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 77 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 78 79 sg = st->sgl; 80 st->nents = 0; 81 sg_page_sizes = 0; 82 for (i = 0; i < page_count; i++) { 83 const unsigned int shrink[] = { 84 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 85 0, 86 }, *s = shrink; 87 gfp_t gfp = noreclaim; 88 89 do { 90 cond_resched(); 91 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 92 if (!IS_ERR(page)) 93 break; 94 95 if (!*s) { 96 ret = PTR_ERR(page); 97 goto err_sg; 98 } 99 100 i915_gem_shrink(i915, 2 * page_count, NULL, *s++); 101 102 /* 103 * We've tried hard to allocate the memory by reaping 104 * our own buffer, now let the real VM do its job and 105 * go down in flames if truly OOM. 106 * 107 * However, since graphics tend to be disposable, 108 * defer the oom here by reporting the ENOMEM back 109 * to userspace. 110 */ 111 if (!*s) { 112 /* reclaim and warn, but no oom */ 113 gfp = mapping_gfp_mask(mapping); 114 115 /* 116 * Our bo are always dirty and so we require 117 * kswapd to reclaim our pages (direct reclaim 118 * does not effectively begin pageout of our 119 * buffers on its own). However, direct reclaim 120 * only waits for kswapd when under allocation 121 * congestion. So as a result __GFP_RECLAIM is 122 * unreliable and fails to actually reclaim our 123 * dirty pages -- unless you try over and over 124 * again with !__GFP_NORETRY. However, we still 125 * want to fail this allocation rather than 126 * trigger the out-of-memory killer and for 127 * this we want __GFP_RETRY_MAYFAIL. 128 */ 129 gfp |= __GFP_RETRY_MAYFAIL; 130 } 131 } while (1); 132 133 if (!i || 134 sg->length >= max_segment || 135 page_to_pfn(page) != last_pfn + 1) { 136 if (i) { 137 sg_page_sizes |= sg->length; 138 sg = sg_next(sg); 139 } 140 st->nents++; 141 sg_set_page(sg, page, PAGE_SIZE, 0); 142 } else { 143 sg->length += PAGE_SIZE; 144 } 145 last_pfn = page_to_pfn(page); 146 147 /* Check that the i965g/gm workaround works. */ 148 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); 149 } 150 if (sg) { /* loop terminated early; short sg table */ 151 sg_page_sizes |= sg->length; 152 sg_mark_end(sg); 153 } 154 155 /* Trim unused sg entries to avoid wasting memory. */ 156 i915_sg_trim(st); 157 158 ret = i915_gem_gtt_prepare_pages(obj, st); 159 if (ret) { 160 /* 161 * DMA remapping failed? One possible cause is that 162 * it could not reserve enough large entries, asking 163 * for PAGE_SIZE chunks instead may be helpful. 164 */ 165 if (max_segment > PAGE_SIZE) { 166 for_each_sgt_page(page, sgt_iter, st) 167 put_page(page); 168 sg_free_table(st); 169 170 max_segment = PAGE_SIZE; 171 goto rebuild_st; 172 } else { 173 dev_warn(&i915->drm.pdev->dev, 174 "Failed to DMA remap %lu pages\n", 175 page_count); 176 goto err_pages; 177 } 178 } 179 180 if (i915_gem_object_needs_bit17_swizzle(obj)) 181 i915_gem_object_do_bit_17_swizzle(obj, st); 182 183 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 184 185 return 0; 186 187 err_sg: 188 sg_mark_end(sg); 189 err_pages: 190 mapping_clear_unevictable(mapping); 191 pagevec_init(&pvec); 192 for_each_sgt_page(page, sgt_iter, st) { 193 if (!pagevec_add(&pvec, page)) 194 check_release_pagevec(&pvec); 195 } 196 if (pagevec_count(&pvec)) 197 check_release_pagevec(&pvec); 198 sg_free_table(st); 199 kfree(st); 200 201 /* 202 * shmemfs first checks if there is enough memory to allocate the page 203 * and reports ENOSPC should there be insufficient, along with the usual 204 * ENOMEM for a genuine allocation failure. 205 * 206 * We use ENOSPC in our driver to mean that we have run out of aperture 207 * space and so want to translate the error from shmemfs back to our 208 * usual understanding of ENOMEM. 209 */ 210 if (ret == -ENOSPC) 211 ret = -ENOMEM; 212 213 return ret; 214 } 215 216 static void 217 shmem_truncate(struct drm_i915_gem_object *obj) 218 { 219 /* 220 * Our goal here is to return as much of the memory as 221 * is possible back to the system as we are called from OOM. 222 * To do this we must instruct the shmfs to drop all of its 223 * backing pages, *now*. 224 */ 225 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 226 obj->mm.madv = __I915_MADV_PURGED; 227 obj->mm.pages = ERR_PTR(-EFAULT); 228 } 229 230 static void 231 shmem_writeback(struct drm_i915_gem_object *obj) 232 { 233 struct address_space *mapping; 234 struct writeback_control wbc = { 235 .sync_mode = WB_SYNC_NONE, 236 .nr_to_write = SWAP_CLUSTER_MAX, 237 .range_start = 0, 238 .range_end = LLONG_MAX, 239 .for_reclaim = 1, 240 }; 241 unsigned long i; 242 243 /* 244 * Leave mmapings intact (GTT will have been revoked on unbinding, 245 * leaving only CPU mmapings around) and add those pages to the LRU 246 * instead of invoking writeback so they are aged and paged out 247 * as normal. 248 */ 249 mapping = obj->base.filp->f_mapping; 250 251 /* Begin writeback on each dirty page */ 252 for (i = 0; i < obj->base.size >> PAGE_SHIFT; i++) { 253 struct page *page; 254 255 page = find_lock_entry(mapping, i); 256 if (!page || xa_is_value(page)) 257 continue; 258 259 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 260 int ret; 261 262 SetPageReclaim(page); 263 ret = mapping->a_ops->writepage(page, &wbc); 264 if (!PageWriteback(page)) 265 ClearPageReclaim(page); 266 if (!ret) 267 goto put; 268 } 269 unlock_page(page); 270 put: 271 put_page(page); 272 } 273 } 274 275 void 276 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 277 struct sg_table *pages, 278 bool needs_clflush) 279 { 280 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 281 282 if (obj->mm.madv == I915_MADV_DONTNEED) 283 obj->mm.dirty = false; 284 285 if (needs_clflush && 286 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 287 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 288 drm_clflush_sg(pages); 289 290 __start_cpu_write(obj); 291 } 292 293 static void 294 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 295 { 296 struct sgt_iter sgt_iter; 297 struct pagevec pvec; 298 struct page *page; 299 300 __i915_gem_object_release_shmem(obj, pages, true); 301 302 i915_gem_gtt_finish_pages(obj, pages); 303 304 if (i915_gem_object_needs_bit17_swizzle(obj)) 305 i915_gem_object_save_bit_17_swizzle(obj, pages); 306 307 mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping); 308 309 pagevec_init(&pvec); 310 for_each_sgt_page(page, sgt_iter, pages) { 311 if (obj->mm.dirty) 312 set_page_dirty(page); 313 314 if (obj->mm.madv == I915_MADV_WILLNEED) 315 mark_page_accessed(page); 316 317 if (!pagevec_add(&pvec, page)) 318 check_release_pagevec(&pvec); 319 } 320 if (pagevec_count(&pvec)) 321 check_release_pagevec(&pvec); 322 obj->mm.dirty = false; 323 324 sg_free_table(pages); 325 kfree(pages); 326 } 327 328 static int 329 shmem_pwrite(struct drm_i915_gem_object *obj, 330 const struct drm_i915_gem_pwrite *arg) 331 { 332 struct address_space *mapping = obj->base.filp->f_mapping; 333 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 334 u64 remain, offset; 335 unsigned int pg; 336 337 /* Caller already validated user args */ 338 GEM_BUG_ON(!access_ok(user_data, arg->size)); 339 340 /* 341 * Before we instantiate/pin the backing store for our use, we 342 * can prepopulate the shmemfs filp efficiently using a write into 343 * the pagecache. We avoid the penalty of instantiating all the 344 * pages, important if the user is just writing to a few and never 345 * uses the object on the GPU, and using a direct write into shmemfs 346 * allows it to avoid the cost of retrieving a page (either swapin 347 * or clearing-before-use) before it is overwritten. 348 */ 349 if (i915_gem_object_has_pages(obj)) 350 return -ENODEV; 351 352 if (obj->mm.madv != I915_MADV_WILLNEED) 353 return -EFAULT; 354 355 /* 356 * Before the pages are instantiated the object is treated as being 357 * in the CPU domain. The pages will be clflushed as required before 358 * use, and we can freely write into the pages directly. If userspace 359 * races pwrite with any other operation; corruption will ensue - 360 * that is userspace's prerogative! 361 */ 362 363 remain = arg->size; 364 offset = arg->offset; 365 pg = offset_in_page(offset); 366 367 do { 368 unsigned int len, unwritten; 369 struct page *page; 370 void *data, *vaddr; 371 int err; 372 char c; 373 374 len = PAGE_SIZE - pg; 375 if (len > remain) 376 len = remain; 377 378 /* Prefault the user page to reduce potential recursion */ 379 err = __get_user(c, user_data); 380 if (err) 381 return err; 382 383 err = __get_user(c, user_data + len - 1); 384 if (err) 385 return err; 386 387 err = pagecache_write_begin(obj->base.filp, mapping, 388 offset, len, 0, 389 &page, &data); 390 if (err < 0) 391 return err; 392 393 vaddr = kmap_atomic(page); 394 unwritten = __copy_from_user_inatomic(vaddr + pg, 395 user_data, 396 len); 397 kunmap_atomic(vaddr); 398 399 err = pagecache_write_end(obj->base.filp, mapping, 400 offset, len, len - unwritten, 401 page, data); 402 if (err < 0) 403 return err; 404 405 /* We don't handle -EFAULT, leave it to the caller to check */ 406 if (unwritten) 407 return -ENODEV; 408 409 remain -= len; 410 user_data += len; 411 offset += len; 412 pg = 0; 413 } while (remain); 414 415 return 0; 416 } 417 418 static void shmem_release(struct drm_i915_gem_object *obj) 419 { 420 fput(obj->base.filp); 421 } 422 423 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 424 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 425 I915_GEM_OBJECT_IS_SHRINKABLE, 426 427 .get_pages = shmem_get_pages, 428 .put_pages = shmem_put_pages, 429 .truncate = shmem_truncate, 430 .writeback = shmem_writeback, 431 432 .pwrite = shmem_pwrite, 433 434 .release = shmem_release, 435 }; 436 437 static int create_shmem(struct drm_i915_private *i915, 438 struct drm_gem_object *obj, 439 size_t size) 440 { 441 unsigned long flags = VM_NORESERVE; 442 struct file *filp; 443 444 drm_gem_private_object_init(&i915->drm, obj, size); 445 446 if (i915->mm.gemfs) 447 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 448 flags); 449 else 450 filp = shmem_file_setup("i915", size, flags); 451 if (IS_ERR(filp)) 452 return PTR_ERR(filp); 453 454 obj->filp = filp; 455 return 0; 456 } 457 458 struct drm_i915_gem_object * 459 i915_gem_object_create_shmem(struct drm_i915_private *i915, u64 size) 460 { 461 struct drm_i915_gem_object *obj; 462 struct address_space *mapping; 463 unsigned int cache_level; 464 gfp_t mask; 465 int ret; 466 467 /* There is a prevalence of the assumption that we fit the object's 468 * page count inside a 32bit _signed_ variable. Let's document this and 469 * catch if we ever need to fix it. In the meantime, if you do spot 470 * such a local variable, please consider fixing! 471 */ 472 if (size >> PAGE_SHIFT > INT_MAX) 473 return ERR_PTR(-E2BIG); 474 475 if (overflows_type(size, obj->base.size)) 476 return ERR_PTR(-E2BIG); 477 478 obj = i915_gem_object_alloc(); 479 if (!obj) 480 return ERR_PTR(-ENOMEM); 481 482 ret = create_shmem(i915, &obj->base, size); 483 if (ret) 484 goto fail; 485 486 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 487 if (IS_I965GM(i915) || IS_I965G(i915)) { 488 /* 965gm cannot relocate objects above 4GiB. */ 489 mask &= ~__GFP_HIGHMEM; 490 mask |= __GFP_DMA32; 491 } 492 493 mapping = obj->base.filp->f_mapping; 494 mapping_set_gfp_mask(mapping, mask); 495 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 496 497 i915_gem_object_init(obj, &i915_gem_shmem_ops); 498 499 obj->write_domain = I915_GEM_DOMAIN_CPU; 500 obj->read_domains = I915_GEM_DOMAIN_CPU; 501 502 if (HAS_LLC(i915)) 503 /* On some devices, we can have the GPU use the LLC (the CPU 504 * cache) for about a 10% performance improvement 505 * compared to uncached. Graphics requests other than 506 * display scanout are coherent with the CPU in 507 * accessing this cache. This means in this mode we 508 * don't need to clflush on the CPU side, and on the 509 * GPU side we only need to flush internal caches to 510 * get data visible to the CPU. 511 * 512 * However, we maintain the display planes as UC, and so 513 * need to rebind when first used as such. 514 */ 515 cache_level = I915_CACHE_LLC; 516 else 517 cache_level = I915_CACHE_NONE; 518 519 i915_gem_object_set_cache_coherency(obj, cache_level); 520 521 trace_i915_gem_object_create(obj); 522 523 return obj; 524 525 fail: 526 i915_gem_object_free(obj); 527 return ERR_PTR(ret); 528 } 529 530 /* Allocate a new GEM object and fill it with the supplied data */ 531 struct drm_i915_gem_object * 532 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 533 const void *data, size_t size) 534 { 535 struct drm_i915_gem_object *obj; 536 struct file *file; 537 size_t offset; 538 int err; 539 540 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 541 if (IS_ERR(obj)) 542 return obj; 543 544 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 545 546 file = obj->base.filp; 547 offset = 0; 548 do { 549 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 550 struct page *page; 551 void *pgdata, *vaddr; 552 553 err = pagecache_write_begin(file, file->f_mapping, 554 offset, len, 0, 555 &page, &pgdata); 556 if (err < 0) 557 goto fail; 558 559 vaddr = kmap(page); 560 memcpy(vaddr, data, len); 561 kunmap(page); 562 563 err = pagecache_write_end(file, file->f_mapping, 564 offset, len, len, 565 page, pgdata); 566 if (err < 0) 567 goto fail; 568 569 size -= len; 570 data += len; 571 offset += len; 572 } while (size); 573 574 return obj; 575 576 fail: 577 i915_gem_object_put(obj); 578 return ERR_PTR(err); 579 } 580